You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/12/11 02:07:49 UTC

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 103 - Still unstable

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/103/

6 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionTest:     1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionTest: 
   1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([CE45C023D1AFBC6B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([CE45C023D1AFBC6B]:0)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
Error from server at http://127.0.0.1:43090: collection already exists: c8n_2x2_commits

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43090: collection already exists: c8n_2x2_commits
	at __randomizedtesting.SeedInfo.seed([CE45C023D1AFBC6B:4611FFF97F53D193]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.BasicDistributedZkTest.createCollection(BasicDistributedZkTest.java:642)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1603)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.multiShardTest(LeaderInitiatedRecoveryOnCommitTest.java:65)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:56)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([CE45C023D1AFBC6B:3708538CEDDAF1E1]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:279)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=2 numReplicas=2 but found DocCollection(solrj_collection//collections/solrj_collection/state.json/34)={   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{     "shard1":{       "range":"80000000-ffffffff",       "state":"active",       "replicas":{         "core_node3":{           "core":"solrj_collection_shard1_replica_n1",           "base_url":"http://127.0.0.1:58487/sl/q",           "node_name":"127.0.0.1:58487_sl%2Fq",           "state":"down",           "type":"NRT"},         "core_node4":{           "core":"solrj_collection_shard1_replica_n2",           "base_url":"http://127.0.0.1:39756/sl/q",           "node_name":"127.0.0.1:39756_sl%2Fq",           "state":"down",           "type":"NRT"}}},     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node7":{           "dataDir":"hdfs://localhost:47566/solr_hdfs_home/solrj_collection/core_node7/data/",           "base_url":"http://127.0.0.1:39756/sl/q",           "node_name":"127.0.0.1:39756_sl%2Fq",           "type":"NRT",           "ulogDir":"hdfs://localhost:47566/solr_hdfs_home/solrj_collection/core_node7/data/tlog",           "core":"solrj_collection_shard2_replica_n6",           "shared_storage":"true",           "state":"active"},         "core_node8":{           "dataDir":"hdfs://localhost:47566/solr_hdfs_home/solrj_collection/core_node8/data/",           "base_url":"http://127.0.0.1:39756/sl/q",           "node_name":"127.0.0.1:39756_sl%2Fq",           "type":"NRT",           "ulogDir":"hdfs://localhost:47566/solr_hdfs_home/solrj_collection/core_node8/data/tlog",           "core":"solrj_collection_shard2_replica_n5",           "shared_storage":"true",           "state":"active",           "leader":"true"}}}},   "router":{     "field":"myOwnField",     "name":"compositeId"},   "maxShardsPerNode":"2",   "autoAddReplicas":"true",   "nrtReplicas":"2",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:39756_sl%2Fq, 127.0.0.1:40735_sl%2Fq, 127.0.0.1:45403_sl%2Fq, 127.0.0.1:51306_sl%2Fq, 127.0.0.1:58487_sl%2Fq]

Stack Trace:
java.lang.AssertionError: Expected numSlices=2 numReplicas=2 but found DocCollection(solrj_collection//collections/solrj_collection/state.json/34)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"solrj_collection_shard1_replica_n1",
          "base_url":"http://127.0.0.1:58487/sl/q",
          "node_name":"127.0.0.1:58487_sl%2Fq",
          "state":"down",
          "type":"NRT"},
        "core_node4":{
          "core":"solrj_collection_shard1_replica_n2",
          "base_url":"http://127.0.0.1:39756/sl/q",
          "node_name":"127.0.0.1:39756_sl%2Fq",
          "state":"down",
          "type":"NRT"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          "dataDir":"hdfs://localhost:47566/solr_hdfs_home/solrj_collection/core_node7/data/",
          "base_url":"http://127.0.0.1:39756/sl/q",
          "node_name":"127.0.0.1:39756_sl%2Fq",
          "type":"NRT",
          "ulogDir":"hdfs://localhost:47566/solr_hdfs_home/solrj_collection/core_node7/data/tlog",
          "core":"solrj_collection_shard2_replica_n6",
          "shared_storage":"true",
          "state":"active"},
        "core_node8":{
          "dataDir":"hdfs://localhost:47566/solr_hdfs_home/solrj_collection/core_node8/data/",
          "base_url":"http://127.0.0.1:39756/sl/q",
          "node_name":"127.0.0.1:39756_sl%2Fq",
          "type":"NRT",
          "ulogDir":"hdfs://localhost:47566/solr_hdfs_home/solrj_collection/core_node8/data/tlog",
          "core":"solrj_collection_shard2_replica_n5",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}}},
  "router":{
    "field":"myOwnField",
    "name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"true",
  "nrtReplicas":"2",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:39756_sl%2Fq, 127.0.0.1:40735_sl%2Fq, 127.0.0.1:45403_sl%2Fq, 127.0.0.1:51306_sl%2Fq, 127.0.0.1:58487_sl%2Fq]
	at __randomizedtesting.SeedInfo.seed([CE45C023D1AFBC6B:4611FFF97F53D193]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:413)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:254)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:143)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud.testRandomUpdates

Error Message:
Could not load collection from ZK: test_col

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: test_col
	at __randomizedtesting.SeedInfo.seed([CE45C023D1AFBC6B:F2841593A6D24279]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1123)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1205)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:848)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud.testRandomUpdates(TestTolerantUpdateProcessorRandomCloud.java:274)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/test_col/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:339)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1135)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1121)
	... 47 more




Build Log:
[...truncated 13153 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestTolerantUpdateProcessorRandomCloud
   [junit4]   2> 1661491 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/init-core-data-001
   [junit4]   2> 1661492 WARN  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1661512 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1661513 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-9182 - causes OOM)
   [junit4]   2> 1661514 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.c.TestTolerantUpdateProcessorRandomCloud Configuring cluster: servers=10, shards=3, repfactor=3
   [junit4]   2> 1661514 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 10 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001
   [junit4]   2> 1661547 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1661579 INFO  (Thread-1467) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1661579 INFO  (Thread-1467) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1661665 ERROR (Thread-1467) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1661703 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:38673
   [junit4]   2> 1662695 INFO  (zkConnectionManagerCallback-525-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1662701 INFO  (jetty-launcher-522-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662701 INFO  (jetty-launcher-522-thread-6) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662702 INFO  (jetty-launcher-522-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662702 INFO  (jetty-launcher-522-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662702 INFO  (jetty-launcher-522-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662717 INFO  (jetty-launcher-522-thread-9) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662717 INFO  (jetty-launcher-522-thread-8) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662735 INFO  (jetty-launcher-522-thread-10) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662736 INFO  (jetty-launcher-522-thread-7) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1662740 INFO  (jetty-launcher-522-thread-10) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4fba237d{/solr,null,AVAILABLE}
   [junit4]   2> 1662741 INFO  (jetty-launcher-522-thread-10) [    ] o.e.j.s.AbstractConnector Started ServerConnector@60a7e60b{HTTP/1.1,[http/1.1]}{127.0.0.1:42518}
   [junit4]   2> 1662741 INFO  (jetty-launcher-522-thread-10) [    ] o.e.j.s.Server Started @1683715ms
   [junit4]   2> 1662741 INFO  (jetty-launcher-522-thread-10) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42518}
   [junit4]   2> 1662741 ERROR (jetty-launcher-522-thread-10) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1662741 INFO  (jetty-launcher-522-thread-10) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1662741 INFO  (jetty-launcher-522-thread-10) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1662741 INFO  (jetty-launcher-522-thread-10) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1662741 INFO  (jetty-launcher-522-thread-10) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:10:59.802Z
   [junit4]   2> 1662760 INFO  (jetty-launcher-522-thread-8) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@444f9fd1{/solr,null,AVAILABLE}
   [junit4]   2> 1662760 INFO  (jetty-launcher-522-thread-8) [    ] o.e.j.s.AbstractConnector Started ServerConnector@71d06cb3{HTTP/1.1,[http/1.1]}{127.0.0.1:47550}
   [junit4]   2> 1662760 INFO  (jetty-launcher-522-thread-8) [    ] o.e.j.s.Server Started @1683734ms
   [junit4]   2> 1662760 INFO  (jetty-launcher-522-thread-8) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47550}
   [junit4]   2> 1662760 ERROR (jetty-launcher-522-thread-8) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1662760 INFO  (jetty-launcher-522-thread-8) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1662760 INFO  (jetty-launcher-522-thread-8) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1662760 INFO  (jetty-launcher-522-thread-8) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1662760 INFO  (jetty-launcher-522-thread-8) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:10:59.821Z
   [junit4]   2> 1662788 INFO  (jetty-launcher-522-thread-9) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ee6127f{/solr,null,AVAILABLE}
   [junit4]   2> 1662788 INFO  (jetty-launcher-522-thread-9) [    ] o.e.j.s.AbstractConnector Started ServerConnector@37a69e2{HTTP/1.1,[http/1.1]}{127.0.0.1:59483}
   [junit4]   2> 1662788 INFO  (jetty-launcher-522-thread-9) [    ] o.e.j.s.Server Started @1683762ms
   [junit4]   2> 1662788 INFO  (jetty-launcher-522-thread-9) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59483}
   [junit4]   2> 1662788 ERROR (jetty-launcher-522-thread-9) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1662788 INFO  (jetty-launcher-522-thread-9) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1662788 INFO  (jetty-launcher-522-thread-9) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1662788 INFO  (jetty-launcher-522-thread-9) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1662788 INFO  (jetty-launcher-522-thread-9) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:10:59.849Z
   [junit4]   2> 1663712 INFO  (jetty-launcher-522-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@35e4045d{/solr,null,AVAILABLE}
   [junit4]   2> 1663712 INFO  (jetty-launcher-522-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@652af336{HTTP/1.1,[http/1.1]}{127.0.0.1:52948}
   [junit4]   2> 1663712 INFO  (jetty-launcher-522-thread-1) [    ] o.e.j.s.Server Started @1684686ms
   [junit4]   2> 1663712 INFO  (jetty-launcher-522-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52948}
   [junit4]   2> 1663712 ERROR (jetty-launcher-522-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1663712 INFO  (jetty-launcher-522-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1663712 INFO  (jetty-launcher-522-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1663712 INFO  (jetty-launcher-522-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664072 INFO  (jetty-launcher-522-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@11e210a1{/solr,null,AVAILABLE}
   [junit4]   2> 1664072 INFO  (jetty-launcher-522-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@55646af7{HTTP/1.1,[http/1.1]}{127.0.0.1:36296}
   [junit4]   2> 1664072 INFO  (jetty-launcher-522-thread-2) [    ] o.e.j.s.Server Started @1685046ms
   [junit4]   2> 1664072 INFO  (jetty-launcher-522-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36296}
   [junit4]   2> 1664072 ERROR (jetty-launcher-522-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1664072 INFO  (jetty-launcher-522-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1664072 INFO  (jetty-launcher-522-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1664073 INFO  (jetty-launcher-522-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664073 INFO  (jetty-launcher-522-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:11:01.134Z
   [junit4]   2> 1664091 INFO  (jetty-launcher-522-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:11:00.774Z
   [junit4]   2> 1664096 INFO  (jetty-launcher-522-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7708c132{/solr,null,AVAILABLE}
   [junit4]   2> 1664096 INFO  (jetty-launcher-522-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5f89cf7f{HTTP/1.1,[http/1.1]}{127.0.0.1:53525}
   [junit4]   2> 1664096 INFO  (jetty-launcher-522-thread-3) [    ] o.e.j.s.Server Started @1685070ms
   [junit4]   2> 1664096 INFO  (jetty-launcher-522-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53525}
   [junit4]   2> 1664096 ERROR (jetty-launcher-522-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1664096 INFO  (jetty-launcher-522-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1664096 INFO  (jetty-launcher-522-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1664096 INFO  (jetty-launcher-522-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664096 INFO  (jetty-launcher-522-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:11:01.157Z
   [junit4]   2> 1664142 INFO  (jetty-launcher-522-thread-6) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@14d8764f{/solr,null,AVAILABLE}
   [junit4]   2> 1664142 INFO  (jetty-launcher-522-thread-6) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3f26a266{HTTP/1.1,[http/1.1]}{127.0.0.1:43964}
   [junit4]   2> 1664142 INFO  (jetty-launcher-522-thread-6) [    ] o.e.j.s.Server Started @1685116ms
   [junit4]   2> 1664142 INFO  (jetty-launcher-522-thread-6) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43964}
   [junit4]   2> 1664142 ERROR (jetty-launcher-522-thread-6) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1664142 INFO  (jetty-launcher-522-thread-6) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1664142 INFO  (jetty-launcher-522-thread-6) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1664142 INFO  (jetty-launcher-522-thread-6) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664142 INFO  (jetty-launcher-522-thread-6) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:11:01.203Z
   [junit4]   2> 1664144 INFO  (jetty-launcher-522-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2ee21f85{/solr,null,AVAILABLE}
   [junit4]   2> 1664144 INFO  (jetty-launcher-522-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1cf4a316{HTTP/1.1,[http/1.1]}{127.0.0.1:38770}
   [junit4]   2> 1664144 INFO  (jetty-launcher-522-thread-4) [    ] o.e.j.s.Server Started @1685118ms
   [junit4]   2> 1664144 INFO  (jetty-launcher-522-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38770}
   [junit4]   2> 1664144 ERROR (jetty-launcher-522-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1664144 INFO  (jetty-launcher-522-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1664144 INFO  (jetty-launcher-522-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1664144 INFO  (jetty-launcher-522-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664144 INFO  (jetty-launcher-522-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:11:01.205Z
   [junit4]   2> 1664323 INFO  (jetty-launcher-522-thread-5) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1664327 INFO  (jetty-launcher-522-thread-5) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@602cc095{/solr,null,AVAILABLE}
   [junit4]   2> 1664328 INFO  (jetty-launcher-522-thread-5) [    ] o.e.j.s.AbstractConnector Started ServerConnector@125d1e36{HTTP/1.1,[http/1.1]}{127.0.0.1:52174}
   [junit4]   2> 1664328 INFO  (jetty-launcher-522-thread-5) [    ] o.e.j.s.Server Started @1685301ms
   [junit4]   2> 1664328 INFO  (jetty-launcher-522-thread-5) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52174}
   [junit4]   2> 1664328 ERROR (jetty-launcher-522-thread-5) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1664328 INFO  (jetty-launcher-522-thread-5) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1664328 INFO  (jetty-launcher-522-thread-5) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1664328 INFO  (jetty-launcher-522-thread-5) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664328 INFO  (jetty-launcher-522-thread-5) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:11:01.389Z
   [junit4]   2> 1664347 INFO  (zkConnectionManagerCallback-541-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1664392 INFO  (jetty-launcher-522-thread-7) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@48085d95{/solr,null,AVAILABLE}
   [junit4]   2> 1664393 INFO  (jetty-launcher-522-thread-7) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3c94bc0c{HTTP/1.1,[http/1.1]}{127.0.0.1:47799}
   [junit4]   2> 1664393 INFO  (jetty-launcher-522-thread-7) [    ] o.e.j.s.Server Started @1685366ms
   [junit4]   2> 1664393 INFO  (jetty-launcher-522-thread-7) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47799}
   [junit4]   2> 1664393 ERROR (jetty-launcher-522-thread-7) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1664393 INFO  (jetty-launcher-522-thread-7) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1664393 INFO  (jetty-launcher-522-thread-7) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1664393 INFO  (jetty-launcher-522-thread-7) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664393 INFO  (jetty-launcher-522-thread-7) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-10T22:11:01.454Z
   [junit4]   2> 1664487 INFO  (jetty-launcher-522-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1664495 INFO  (zkConnectionManagerCallback-527-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1664495 INFO  (zkConnectionManagerCallback-529-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1664495 INFO  (zkConnectionManagerCallback-531-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1664573 INFO  (zkConnectionManagerCallback-533-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1664589 INFO  (jetty-launcher-522-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1664607 INFO  (jetty-launcher-522-thread-8) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1664607 INFO  (jetty-launcher-522-thread-10) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1665349 INFO  (jetty-launcher-522-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1665355 INFO  (zkConnectionManagerCallback-543-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1666024 INFO  (jetty-launcher-522-thread-9) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1666032 INFO  (jetty-launcher-522-thread-5) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1666035 INFO  (zkConnectionManagerCallback-539-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1666067 INFO  (jetty-launcher-522-thread-9) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1666079 INFO  (zkConnectionManagerCallback-545-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1666432 INFO  (jetty-launcher-522-thread-8) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1666433 INFO  (jetty-launcher-522-thread-6) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1666475 INFO  (zkConnectionManagerCallback-553-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1666379 INFO  (zkConnectionManagerCallback-535-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1666509 INFO  (jetty-launcher-522-thread-6) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1666511 INFO  (jetty-launcher-522-thread-10) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1666377 INFO  (zkConnectionManagerCallback-549-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1666375 INFO  (zkConnectionManagerCallback-537-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1666527 INFO  (zkConnectionManagerCallback-567-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1666517 INFO  (jetty-launcher-522-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1666516 INFO  (jetty-launcher-522-thread-7) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1666576 INFO  (jetty-launcher-522-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1666576 INFO  (jetty-launcher-522-thread-5) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1667665 INFO  (zkConnectionManagerCallback-560-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1667740 INFO  (jetty-launcher-522-thread-7) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1667741 INFO  (jetty-launcher-522-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1667665 INFO  (zkConnectionManagerCallback-571-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1667771 INFO  (zkConnectionManagerCallback-569-thread-1-processing-n:127.0.0.1:59483_solr) [n:127.0.0.1:59483_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1667741 INFO  (jetty-launcher-522-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1668014 INFO  (zkConnectionManagerCallback-573-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668017 INFO  (zkConnectionManagerCallback-575-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668075 INFO  (jetty-launcher-522-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38673/solr
   [junit4]   2> 1668227 INFO  (zkConnectionManagerCallback-585-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668286 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x160427b33d10010, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1668287 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x160427b33d10011, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1668287 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x160427b33d10012, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1668255 INFO  (zkConnectionManagerCallback-583-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668275 INFO  (zkConnectionManagerCallback-581-thread-1-processing-n:127.0.0.1:38770_solr) [n:127.0.0.1:38770_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668407 INFO  (zkConnectionManagerCallback-587-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668450 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x160427b33d10013, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1668450 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x160427b33d10016, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1668502 INFO  (jetty-launcher-522-thread-6) [n:127.0.0.1:43964_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1668502 INFO  (jetty-launcher-522-thread-4) [n:127.0.0.1:38770_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1668503 INFO  (jetty-launcher-522-thread-6) [n:127.0.0.1:43964_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43964_solr
   [junit4]   2> 1668503 INFO  (jetty-launcher-522-thread-4) [n:127.0.0.1:38770_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38770_solr
   [junit4]   2> 1668504 INFO  (jetty-launcher-522-thread-6) [n:127.0.0.1:43964_solr    ] o.a.s.c.Overseer Overseer (id=99152288719896596-127.0.0.1:43964_solr-n_0000000000) starting
   [junit4]   2> 1668521 INFO  (zkConnectionManagerCallback-589-thread-1-processing-n:127.0.0.1:47550_solr) [n:127.0.0.1:47550_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668755 INFO  (zkConnectionManagerCallback-593-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668761 INFO  (jetty-launcher-522-thread-8) [n:127.0.0.1:47550_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1668775 INFO  (jetty-launcher-522-thread-8) [n:127.0.0.1:47550_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1669191 INFO  (jetty-launcher-522-thread-8) [n:127.0.0.1:47550_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47550_solr
   [junit4]   2> 1670507 INFO  (zkConnectionManagerCallback-598-thread-1-processing-n:127.0.0.1:42518_solr) [n:127.0.0.1:42518_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1670508 INFO  (zkConnectionManagerCallback-596-thread-1-processing-n:127.0.0.1:36296_solr) [n:127.0.0.1:36296_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1670603 INFO  (zkConnectionManagerCallback-600-thread-1-processing-n:127.0.0.1:52174_solr) [n:127.0.0.1:52174_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1670605 INFO  (jetty-launcher-522-thread-9) [n:127.0.0.1:59483_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1670612 INFO  (jetty-launcher-522-thread-9) [n:127.0.0.1:59483_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1670648 INFO  (jetty-launcher-522-thread-9) [n:127.0.0.1:59483_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59483_solr
   [junit4]   2> 1670925 INFO  (jetty-launcher-522-thread-4) [n:127.0.0.1:38770_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38770.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1671358 INFO  (zkCallback-582-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1671400 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x160427b33d10017, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1671529 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
   [junit4]   2> 1671529 INFO  (jetty-launcher-522-thread-6) [n:127.0.0.1:43964_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43964_solr
   [junit4]   2> 1671588 INFO  (zkCallback-580-thread-2-processing-n:127.0.0.1:38770_solr) [n:127.0.0.1:38770_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1671612 INFO  (jetty-launcher-522-thread-4) [n:127.0.0.1:38770_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38770.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1671631 INFO  (zkCallback-568-thread-1-processing-n:127.0.0.1:59483_solr) [n:127.0.0.1:59483_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1671651 INFO  (jetty-launcher-522-thread-2) [n:127.0.0.1:36296_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1671653 INFO  (jetty-launcher-522-thread-2) [n:127.0.0.1:36296_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1671653 INFO  (jetty-launcher-522-thread-4) [n:127.0.0.1:38770_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38770.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1671671 INFO  (zkCallback-582-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1671672 INFO  (jetty-launcher-522-thread-2) [n:127.0.0.1:36296_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36296_solr
   [junit4]   2> 1671683 INFO  (jetty-launcher-522-thread-4) [n:127.0.0.1:38770_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node2/.
   [junit4]   2> 1671703 INFO  (zkCallback-588-thread-1-processing-n:127.0.0.1:47550_solr) [n:127.0.0.1:47550_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (5)
   [junit4]   2> 1671728 INFO  (jetty-launcher-522-thread-5) [n:127.0.0.1:52174_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1671728 INFO  (jetty-launcher-522-thread-10) [n:127.0.0.1:42518_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1671808 INFO  (jetty-launcher-522-thread-10) [n:127.0.0.1:42518_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1671993 INFO  (jetty-launcher-522-thread-5) [n:127.0.0.1:52174_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1672014 INFO  (zkCallback-568-thread-1-processing-n:127.0.0.1:59483_solr) [n:127.0.0.1:59483_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (5)
   [junit4]   2> 1672015 INFO  (zkCallback-582-thread-2-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1672015 INFO  (zkCallback-580-thread-1-processing-n:127.0.0.1:38770_solr) [n:127.0.0.1:38770_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1672051 INFO  (zkConnectionManagerCallback-606-thread-1-processing-n:127.0.0.1:52948_solr) [n:127.0.0.1:52948_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1672115 INFO  (jetty-launcher-522-thread-1) [n:127.0.0.1:52948_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1672116 INFO  (jetty-launcher-522-thread-1) [n:127.0.0.1:52948_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1672132 INFO  (jetty-launcher-522-thread-1) [n:127.0.0.1:52948_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52948_solr
   [junit4]   2> 1672134 INFO  (zkCallback-580-thread-1-processing-n:127.0.0.1:38770_solr) [n:127.0.0.1:38770_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1672183 INFO  (zkConnectionManagerCallback-604-thread-1-processing-n:127.0.0.1:47799_solr) [n:127.0.0.1:47799_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1672232 INFO  (jetty-launcher-522-thread-10) [n:127.0.0.1:42518_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42518_solr
   [junit4]   2> 1672233 INFO  (jetty-launcher-522-thread-5) [n:127.0.0.1:52174_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52174_solr
   [junit4]   2> 1672254 INFO  (zkCallback-580-thread-1-processing-n:127.0.0.1:38770_solr) [n:127.0.0.1:38770_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (8)
   [junit4]   2> 1672573 INFO  (zkCallback-588-thread-1-processing-n:127.0.0.1:47550_solr) [n:127.0.0.1:47550_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (8)
   [junit4]   2> 1672573 INFO  (zkCallback-582-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (8)
   [junit4]   2> 1672573 INFO  (zkCallback-568-thread-2-processing-n:127.0.0.1:59483_solr) [n:127.0.0.1:59483_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (8)
   [junit4]   2> 1672577 INFO  (jetty-launcher-522-thread-7) [n:127.0.0.1:47799_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (8)
   [junit4]   2> 1672578 INFO  (jetty-launcher-522-thread-7) [n:127.0.0.1:47799_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1672580 INFO  (jetty-launcher-522-thread-7) [n:127.0.0.1:47799_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47799_solr
   [junit4]   2> 1672581 INFO  (zkCallback-588-thread-1-processing-n:127.0.0.1:47550_solr) [n:127.0.0.1:47550_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
   [junit4]   2> 1672581 INFO  (zkCallback-568-thread-2-processing-n:127.0.0.1:59483_solr) [n:127.0.0.1:59483_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
   [junit4]   2> 1672581 INFO  (zkCallback-582-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
   [junit4]   2> 1672587 INFO  (zkCallback-599-thread-1-processing-n:127.0.0.1:52174_solr) [n:127.0.0.1:52174_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (8)
   [junit4]   2> 1672590 INFO  (zkCallback-580-thread-1-processing-n:127.0.0.1:38770_solr) [n:127.0.0.1:38770_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
   [junit4]   2> 1672687 INFO  (zkConnectionManagerCallback-610-thread-1-processing-n:127.0.0.1:53525_solr) [n:127.0.0.1:53525_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1672743 INFO  (zkCallback-599-thread-2-processing-n:127.0.0.1:52174_solr) [n:127.0.0.1:52174_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
   [junit4]   2> 1672787 INFO  (zkCallback-595-thread-1-processing-n:127.0.0.1:36296_solr) [n:127.0.0.1:36296_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (9)
   [junit4]   2> 1672788 INFO  (zkCallback-605-thread-1-processing-n:127.0.0.1:52948_solr) [n:127.0.0.1:52948_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (9)
   [junit4]   2> 1672795 INFO  (zkCallback-603-thread-1-processing-n:127.0.0.1:47799_solr) [n:127.0.0.1:47799_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (9)
   [junit4]   2> 1673619 INFO  (jetty-launcher-522-thread-3) [n:127.0.0.1:53525_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (9)
   [junit4]   2> 1673620 INFO  (jetty-launcher-522-thread-3) [n:127.0.0.1:53525_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1673642 INFO  (jetty-launcher-522-thread-3) [n:127.0.0.1:53525_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53525_solr
   [junit4]   2> 1673643 INFO  (zkCallback-588-thread-1-processing-n:127.0.0.1:47550_solr) [n:127.0.0.1:47550_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1673643 INFO  (zkCallback-568-thread-2-processing-n:127.0.0.1:59483_solr) [n:127.0.0.1:59483_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1673643 INFO  (zkCallback-582-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1673663 INFO  (zkCallback-580-thread-1-processing-n:127.0.0.1:38770_solr) [n:127.0.0.1:38770_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1673992 INFO  (zkCallback-597-thread-1-processing-n:127.0.0.1:42518_solr) [n:127.0.0.1:42518_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (9)
   [junit4]   2> 1673995 INFO  (zkCallback-605-thread-1-processing-n:127.0.0.1:52948_solr) [n:127.0.0.1:52948_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1673995 INFO  (zkCallback-603-thread-1-processing-n:127.0.0.1:47799_solr) [n:127.0.0.1:47799_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1674002 INFO  (zkCallback-599-thread-2-processing-n:127.0.0.1:52174_solr) [n:127.0.0.1:52174_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1674012 INFO  (zkCallback-595-thread-1-processing-n:127.0.0.1:36296_solr) [n:127.0.0.1:36296_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1674092 INFO  (zkCallback-609-thread-1-processing-n:127.0.0.1:53525_solr) [n:127.0.0.1:53525_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1674138 INFO  (jetty-launcher-522-thread-9) [n:127.0.0.1:59483_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59483.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674142 INFO  (zkCallback-597-thread-2-processing-n:127.0.0.1:42518_solr) [n:127.0.0.1:42518_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (9) -> (10)
   [junit4]   2> 1674161 INFO  (jetty-launcher-522-thread-8) [n:127.0.0.1:47550_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47550.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674174 INFO  (jetty-launcher-522-thread-1) [n:127.0.0.1:52948_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52948.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674213 INFO  (jetty-launcher-522-thread-6) [n:127.0.0.1:43964_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43964.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674254 INFO  (jetty-launcher-522-thread-1) [n:127.0.0.1:52948_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52948.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674255 INFO  (jetty-launcher-522-thread-1) [n:127.0.0.1:52948_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52948.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674260 INFO  (jetty-launcher-522-thread-8) [n:127.0.0.1:47550_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47550.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674260 INFO  (jetty-launcher-522-thread-8) [n:127.0.0.1:47550_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47550.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674261 INFO  (jetty-launcher-522-thread-8) [n:127.0.0.1:47550_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node8/.
   [junit4]   2> 1674288 INFO  (jetty-launcher-522-thread-9) [n:127.0.0.1:59483_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59483.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674288 INFO  (jetty-launcher-522-thread-9) [n:127.0.0.1:59483_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59483.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674290 INFO  (jetty-launcher-522-thread-6) [n:127.0.0.1:43964_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43964.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674290 INFO  (jetty-launcher-522-thread-6) [n:127.0.0.1:43964_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43964.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674475 INFO  (jetty-launcher-522-thread-6) [n:127.0.0.1:43964_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node3/.
   [junit4]   2> 1674475 INFO  (jetty-launcher-522-thread-1) [n:127.0.0.1:52948_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node6/.
   [junit4]   2> 1674532 INFO  (jetty-launcher-522-thread-9) [n:127.0.0.1:59483_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node7/.
   [junit4]   2> 1674575 INFO  (jetty-launcher-522-thread-2) [n:127.0.0.1:36296_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36296.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674905 INFO  (jetty-launcher-522-thread-2) [n:127.0.0.1:36296_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36296.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674906 INFO  (jetty-launcher-522-thread-2) [n:127.0.0.1:36296_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36296.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1674909 INFO  (jetty-launcher-522-thread-2) [n:127.0.0.1:36296_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node5/.
   [junit4]   2> 1674917 INFO  (jetty-launcher-522-thread-10) [n:127.0.0.1:42518_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42518.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675098 INFO  (jetty-launcher-522-thread-7) [n:127.0.0.1:47799_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47799.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675120 INFO  (jetty-launcher-522-thread-10) [n:127.0.0.1:42518_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42518.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675121 INFO  (jetty-launcher-522-thread-10) [n:127.0.0.1:42518_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42518.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675155 INFO  (jetty-launcher-522-thread-10) [n:127.0.0.1:42518_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node9/.
   [junit4]   2> 1675414 INFO  (jetty-launcher-522-thread-7) [n:127.0.0.1:47799_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47799.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675415 INFO  (jetty-launcher-522-thread-7) [n:127.0.0.1:47799_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47799.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675435 INFO  (jetty-launcher-522-thread-7) [n:127.0.0.1:47799_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node10/.
   [junit4]   2> 1675450 INFO  (jetty-launcher-522-thread-3) [n:127.0.0.1:53525_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53525.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675471 INFO  (jetty-launcher-522-thread-5) [n:127.0.0.1:52174_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52174.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675532 INFO  (jetty-launcher-522-thread-5) [n:127.0.0.1:52174_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52174.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675532 INFO  (jetty-launcher-522-thread-5) [n:127.0.0.1:52174_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_52174.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675541 INFO  (jetty-launcher-522-thread-3) [n:127.0.0.1:53525_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53525.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675541 INFO  (jetty-launcher-522-thread-3) [n:127.0.0.1:53525_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53525.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c76bfb5
   [junit4]   2> 1675570 INFO  (jetty-launcher-522-thread-3) [n:127.0.0.1:53525_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node4/.
   [junit4]   2> 1675570 INFO  (jetty-launcher-522-thread-5) [n:127.0.0.1:52174_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestTolerantUpdateProcessorRandomCloud_CE45C023D1AFBC6B-001/tempDir-001/node1/.
   [junit4]   2> 1680759 INFO  (zkConnectionManagerCallback-618-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1680811 INFO  (zkConnectionManagerCallback-622-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1680905 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (10)
   [junit4]   2> 1680906 INFO  (SUITE-TestTolerantUpdateProcessorRandomCloud-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38673/solr ready
   [junit4]   2> 1683656 INFO  (qtp125331451-3686) [n:127.0.0.1:53525_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=3&property.config=solrconfig-distrib-update-processor-chains.xml&property.schema=schema15.xml&collection.configName=solrCloudCollectionConfig&name=test_col&nrtReplicas=3&action=CREATE&numShards=3&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1683728 INFO  (OverseerThreadFactory-1204-thread-1-processing-n:127.0.0.1:43964_solr) [n:127.0.0.1:43964_solr    ] o.a.s.c.CreateCollectionCmd Create collection test_col
   [junit4]   2> 1683984 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"test_col_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:42518/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1683984 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"test_col_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52948/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1684007 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"test_col_shard1_replica_n5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:52174/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1684044 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"test_col_shard2_replica_n7",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47799/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1684047 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"test_col_shard2_replica_n8",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:43964/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1684061 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"test_col_shard2_replica_n10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:59483/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1684085 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"test_col_shard3_replica_n12",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:53525/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1684096 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"test_col_shard3_replica_n15",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:36296/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1684123 INFO  (OverseerStateUpdate-99152288719896596-127.0.0.1:43964_solr-n_0000000000) [n:127.0.0.1:43964_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"test_col",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"test_col_shard3_replica_n16",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:47550/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1684764 INFO  (qtp125331451-3684) [n:127.0.0.1:53525_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=test_col&version=2&replicaType=NRT&property.config=solrconfig-distrib-update-processor-chains.xml&property.schema=schema15.xml&coreNodeName=core_node14&name=test_col_shard3_replica_n12&action=CREATE&numShards=3&shard=shard3&wt=javabin
   [junit4]   2> 1684765 INFO  (qtp125331451-3684) [n:127.0.0.1:53525_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1684876 INFO  (zkCallback-609-thread-1-processing-n:127.0.0.1:53525_solr) [n:127.0.0.1:53525_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test_col/state.json] for collection [test_col] has occurred - updating... (live nodes size: [10])
   [junit4]   2> 1684956 INFO  (zkCallback-609-thread-2-processing-n:127.0.0.1:53525_solr) [n:127.0.0.1:53525_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test_col/state.json] for collection [test_col] has occurred - updating... (live nodes size: [10])
   [junit4]   2> 1684992 INFO  (qtp1888143463-3633) [n:127.0.0.1:42518_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=test_col&version=2&replicaType=NRT&property.config=solrconfig-distrib-update-processor-chains.xml&property.schema=schema15.xml&coreNodeName=core_node3&name=test_col_shard1_replica_n1&action=CREATE&numShards=3&shard=shard1&wt=javabin
   [junit4]   2> 1684999 INFO  (qtp1888143463-3633) [n:127.0.0.1:42518_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1685074 INFO  (qtp1700367677-3666) [n:127.0.0.1:52948_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=test_col&version=2&replicaType=NRT&property.config=solrconfig-distrib-update-processor-chains.xml&property.schema=schema15.xml&coreNodeName=core_node4&name=test_col_shard1_replica_n2&action=CREATE&numShards=3&shard=shard1&wt=javabin
   [junit4]   2> 1685079 INFO  (qtp1843495998-3648) [n:127.0.0.1:47550_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=test_col&version=2&replicaType=NRT&property.config=solrconfig-distrib-update-processor-chains.xml&property.schema=schema15.xml&coreNodeName=core_node18&name=test_col_shard3_replica_n16&action=CREATE&numShards=3&shard=shard3&wt=javabin
   [junit4]   2> 1685080 INFO  (qtp1843495998-3648) [n:127.0.0.1:47550_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1685080 INFO  (qtp513584837-3728) [n:127.0.0.1:47799_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=test_col&version=2&replicaType=NRT&property.config=solrconfig-distrib-update-processor-chains.xml&property.schema=schema15.xml&coreNodeName=core_node9&name=test_col_shard2_replica_n7&action=CREATE&numShards=3&shard=shard2&wt=javabin
   [junit4]   2> 1685080 INFO  (qtp513584837-3728) [n:127.0.0.1:47799_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1685081 INFO  (qtp49968675-3715) [n:127.0.0.1:52174_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=solrCloudCollectionConfig&newCollection=true&collection=test_col&version=2&replicaType=NRT&property.config=solrconfig-distrib-update-processor-chains.xml&property.schema=schema15.xml&coreNodeName=core_node6&name=test_col_shard1_replica_n5&action=CREATE&numShards=3&shard=shard1&wt=javabin
   [junit4]   2> 1685081 INFO  (qtp49968675-3715) [n:127.0.0.1:52174_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1685089 INFO  (qtp1700367677-3666) [n:127.0.0.1:52948_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1685227 INFO  (zkCallback-597-thread-2-processing-n:127.0.0.1:42518_solr) [n:127.0.0.1:42518_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test_col/state.json] for collection [test_col] has occurred - updating... (live nodes size: [10])
   [junit4]   2> 1685227 INFO  (zkCallback-599-thread-2-processing-n:127.0.0.1:52174_solr) [n:127.0.0.1:52174_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test_col/state.json] for collection [test_col] has occurred - updating... (live nodes size: [10])
   [junit4]   2> 1685227 INFO  (zkCallback-605-thread-1-processing-n:127.0.0.1:52948_solr) [n:127.0.0.1:52948_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test_col/state.json] for collection [test_col] has occurred - updating... (live nodes size: [10])
   [junit4]   2> 1685227 INFO  (zkCallback-588-thread-1-processing-n:127.0.0.1:47550_solr) [n:127.0.0.1:47550_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/test_col/state.json] for collection [test_col] has occurred - updating... (live nodes size: [10])
   [junit4]   2> 1685227 INFO  (zkCallback-603-thread-1-processing-n:127.0.0.1:4779

[...truncated too long message...]

testStressElection-seed#[CE45C023D1AFBC6B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStressElection
   [junit4]   2> 6183276 INFO  (TEST-LeaderElectionTest.testBasic-seed#[CE45C023D1AFBC6B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBasic
   [junit4]   2> 6183276 INFO  (TEST-LeaderElectionTest.testBasic-seed#[CE45C023D1AFBC6B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 6183316 INFO  (Thread-4239) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 6183316 INFO  (Thread-4239) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 6183387 ERROR (Thread-4239) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 6183419 INFO  (TEST-LeaderElectionTest.testBasic-seed#[CE45C023D1AFBC6B]) [    ] o.a.s.c.ZkTestServer start zk server on port:35778
   [junit4]   2> 6183436 INFO  (zkConnectionManagerCallback-1466-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6183504 INFO  (zkConnectionManagerCallback-1468-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6183852 INFO  (zkConnectionManagerCallback-1470-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6183862 ERROR (TEST-LeaderElectionTest.testBasic-seed#[CE45C023D1AFBC6B]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 6183862 INFO  (TEST-LeaderElectionTest.testBasic-seed#[CE45C023D1AFBC6B]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35778 35778
   [junit4]   2> 6184019 INFO  (Thread-4239) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35778 35778
   [junit4]   2> 6184101 INFO  (TEST-LeaderElectionTest.testBasic-seed#[CE45C023D1AFBC6B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBasic
   [junit4]   2> 6184102 INFO  (SUITE-LeaderElectionTest-seed#[CE45C023D1AFBC6B]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Dec 10, 2017 11:26:21 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Dec 10, 2017 11:26:41 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionTest: 
   [junit4]   2>    1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Dec 10, 2017 11:26:41 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]
   [junit4]   2> Dec 10, 2017 11:26:44 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=798, maxMBSortInHeap=7.489695383513727, sim=RandomSimilarity(queryNorm=false): {}, locale=sr-BA, timezone=Africa/Nouakchott
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=219828376,total=519045120
   [junit4]   2> NOTE: All tests run in this JVM: [TestLMDirichletSimilarityFactory, TestInitParams, MetricsHandlerTest, TestFreeTextSuggestions, NodeMutatorTest, TestLRUCache, CollectionReloadTest, XmlUpdateRequestHandlerTest, TestFieldResource, HighlighterMaxOffsetTest, TestTrie, TestPivotHelperCode, RequiredFieldsTest, TestLeaderInitiatedRecoveryThread, TestMiniSolrCloudClusterSSL, CoreAdminHandlerTest, TestReloadDeadlock, TestValueSourceCache, JvmMetricsTest, SegmentsInfoRequestHandlerTest, HdfsRecoveryZkTest, TestCloudSearcherWarming, ConfigSetsAPITest, ZkNodePropsTest, GraphQueryTest, ExplicitHLLTest, TestPayloadScoreQParserPlugin, TestFiltering, TestDistributedMissingSort, BlockDirectoryTest, LargeFieldTest, CSVRequestHandlerTest, IndexSchemaTest, ResponseHeaderTest, TestTolerantUpdateProcessorRandomCloud, TestDistribDocBasedVersion, OverseerCollectionConfigSetProcessorTest, ClusterStateTest, FullSolrCloudDistribCmdsTest, QueryElevationComponentTest, TestChildDocTransformer, TestLegacyField, LoggingHandlerTest, TestOrdValues, CloudMLTQParserTest, SuggesterTest, HdfsThreadLeakTest, TestHalfAndHalfDocValues, TestRealTimeGet, TestReplicationHandlerBackup, TestCharFilters, AutoAddReplicasIntegrationTest, TestHdfsCloudBackupRestore, LeaderInitiatedRecoveryOnCommitTest, TestMaxScoreQueryParser, TestReload, SolrMetricManagerTest, SharedFSAutoReplicaFailoverTest, TestSolrCloudWithSecureImpersonation, RecoveryZkTest, SimplePostToolTest, TestFieldSortValues, TestDistributedGrouping, CollectionsAPIAsyncDistributedZkTest, AlternateDirectoryTest, TestManagedStopFilterFactory, TestSolrCloudWithHadoopAuthPlugin, TestSolrDeletionPolicy2, LeaderElectionTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderElectionTest -Dtests.seed=CE45C023D1AFBC6B -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-BA -Dtests.timezone=Africa/Nouakchott -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | LeaderElectionTest (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionTest: 
   [junit4]    >    1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CE45C023D1AFBC6B]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=11058, name=zkConnectionManagerCallback-1446-thread-1, state=WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CE45C023D1AFBC6B]:0)
   [junit4] Completed [181/751 (5!)] on J2 in 59.19s, 5 tests, 2 errors <<< FAILURES!

[...truncated 49179 lines...]

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 107 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/107/

16 tests failed.
FAILED:  org.apache.lucene.analysis.opennlp.TestOpenNLPPOSFilterFactory.testBasic

Error Message:
term 0 expected:<[Senntennce]> but was:<[.]>

Stack Trace:
org.junit.ComparisonFailure: term 0 expected:<[Senntennce]> but was:<[.]>
	at __randomizedtesting.SeedInfo.seed([48D3596BCDAE3476:E329447E1272B258]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents(BaseTokenStreamTestCase.java:201)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents(BaseTokenStreamTestCase.java:325)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents(BaseTokenStreamTestCase.java:329)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:865)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:727)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:723)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo(BaseTokenStreamTestCase.java:384)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo(BaseTokenStreamTestCase.java:420)
	at org.apache.lucene.analysis.opennlp.TestOpenNLPPOSFilterFactory.testBasic(TestOpenNLPPOSFilterFactory.java:67)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([4C928641B0C06BCD]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexWriterOnVMError

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([4C928641B0C06BCD]:0)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node5:{"core":"c8n_1x3_lf_shard1_replica_n2","base_url":"http://127.0.0.1:41635","node_name":"127.0.0.1:41635_","state":"active","type":"NRT","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/18)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node4":{           "state":"down",           "base_url":"http://127.0.0.1:40791",           "core":"c8n_1x3_lf_shard1_replica_n1",           "node_name":"127.0.0.1:40791_",           "type":"NRT"},         "core_node5":{           "core":"c8n_1x3_lf_shard1_replica_n2",           "base_url":"http://127.0.0.1:41635",           "node_name":"127.0.0.1:41635_",           "state":"active",           "type":"NRT",           "leader":"true"},         "core_node6":{           "core":"c8n_1x3_lf_shard1_replica_n3",           "base_url":"http://127.0.0.1:34725",           "node_name":"127.0.0.1:34725_",           "state":"down",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"3",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node5:{"core":"c8n_1x3_lf_shard1_replica_n2","base_url":"http://127.0.0.1:41635","node_name":"127.0.0.1:41635_","state":"active","type":"NRT","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/18)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "state":"down",
          "base_url":"http://127.0.0.1:40791",
          "core":"c8n_1x3_lf_shard1_replica_n1",
          "node_name":"127.0.0.1:40791_",
          "type":"NRT"},
        "core_node5":{
          "core":"c8n_1x3_lf_shard1_replica_n2",
          "base_url":"http://127.0.0.1:41635",
          "node_name":"127.0.0.1:41635_",
          "state":"active",
          "type":"NRT",
          "leader":"true"},
        "core_node6":{
          "core":"c8n_1x3_lf_shard1_replica_n3",
          "base_url":"http://127.0.0.1:34725",
          "node_name":"127.0.0.1:34725_",
          "state":"down",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:C44D89428589A9EC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:169)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testReadApi

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:1B304D2DF087260F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testReadApi(AutoScalingHandlerTest.java:725)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestGenericDistributedQueue

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414]:0)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState

Error Message:
Two TriggerAction instances should have been created by now

Stack Trace:
java.lang.AssertionError: Two TriggerAction instances should have been created by now
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:C4243FE711B525B9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:361)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventQueue

Error Message:
action wasn't interrupted

Stack Trace:
java.lang.AssertionError: action wasn't interrupted
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:85ACF436221202E1]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventQueue(TestTriggerIntegration.java:638)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([C2908D1F73859111]:0)


FAILED:  org.apache.solr.cloud.UnloadDistributedZkTest.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:C44D89428589A9EC]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1108)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
	at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:137)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:441)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:339)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1120)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1106)
	... 38 more


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:38010_solr wasn't created

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:38010_solr wasn't created
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:54A33E94254009FB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:932)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestGenericDistributedQueue.testDistributedQueue

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414]:0)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37212

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37212
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:C44D89428589A9EC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.BasicDistributedZkTest.createCollection(BasicDistributedZkTest.java:642)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1604)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.multiShardTest(LeaderInitiatedRecoveryOnCommitTest.java:65)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:56)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:37212 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 48 more


FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.test

Error Message:
expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:C44D89428589A9EC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:203)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([C2908D1F73859111]:0)


FAILED:  org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.test

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414:C44D89428589A9EC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.assertInvariants(TimeRoutedAliasUpdateProcessorTest.java:245)
	at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.test(TimeRoutedAliasUpdateProcessorTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 1808 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4]   2> dic 19, 2017 1:30:23 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4]   2>    1) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    2) Thread[id=5332, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[4C928641B0C06BCD], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]   2>         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]   2>         at java.lang.Throwable.getStackTrace(Throwable.java:816)
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError$4.eval(TestIndexWriterOnVMError.java:283)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
   [junit4]   2>         at org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.write(Lucene60FieldInfosFormat.java:286)
   [junit4]   2>         at org.apache.lucene.index.ReadersAndUpdates.writeFieldInfosGen(ReadersAndUpdates.java:642)
   [junit4]   2>         at org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:721)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter$ReaderPool.commit(IndexWriter.java:791)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3266)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3458)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3423)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    3) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    4) Thread[id=5331, name=SUITE-TestIndexWriterOnVMError-seed#[4C928641B0C06BCD], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterOnVMError -Dtests.method=testCheckpoint -Dtests.seed=4C928641B0C06BCD -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-CU -Dtests.timezone=ART -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   7201s J1 | TestIndexWriterOnVMError.testCheckpoint <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4C928641B0C06BCD]:0)
   [junit4]   2> dic 19, 2017 1:30:24 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> dic 19, 2017 1:30:44 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterOnVMError: 
   [junit4]   2>    1) Thread[id=5332, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[4C928641B0C06BCD], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]   2>         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]   2>         at java.lang.Throwable.getStackTrace(Throwable.java:816)
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError$4.eval(TestIndexWriterOnVMError.java:283)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:73)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:107)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5010)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:538)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:491)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:557)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:673)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3246)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3458)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3423)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> dic 19, 2017 1:30:44 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=5332, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[4C928641B0C06BCD], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2> dic 19, 2017 1:30:47 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=5332, name=TEST-TestIndexWriterOnVMError.testCheckpoint-seed#[4C928641B0C06BCD], state=RUNNABLE, group=TGRP-TestIndexWriterOnVMError]
   [junit4]   2>         at java.lang.Throwable.getStackTraceElement(Native Method)
   [junit4]   2>         at java.lang.Throwable.getOurStackTrace(Throwable.java:827)
   [junit4]   2>         at java.lang.Throwable.getStackTrace(Throwable.java:816)
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError$4.eval(TestIndexWriterOnVMError.java:283)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowDeterministicException(MockDirectoryWrapper.java:1022)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:144)
   [junit4]   2>         at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
   [junit4]   2>         at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:72)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70SegmentInfoFormat.write(Lucene70SegmentInfoFormat.java:298)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:547)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:491)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:557)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:673)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3246)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3458)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3423)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.doTest(TestIndexWriterOnVMError.java:181)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterOnVMError.testCheckpoint(TestIndexWriterOnVMError.java:280)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {text_payloads=PostingsFormat(name=LuceneVarGapDocFreqInterval), text_vectors=BlockTreeOrds(blocksize=128), text1=FSTOrd50, id=FST50}, docValues:{dv3=DocValuesFormat(name=Lucene70), dv2=DocValuesFormat(name=Direct), dv5=DocValuesFormat(name=Memory), dv=DocValuesFormat(name=Asserting), text_payloads=DocValuesFormat(name=Lucene70), dv4=DocValuesFormat(name=Asserting), text_vectors=DocValuesFormat(name=Asserting), text1=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Memory), point2d=DocValuesFormat(name=Asserting), stored1=DocValuesFormat(name=Asserting), point=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1608, maxMBSortInHeap=6.202248228411669, sim=RandomSimilarity(queryNorm=false): {text_payloads=IB LL-D3(800.0), text_vectors=DFR I(ne)B1, text1=DFR I(ne)1}, locale=es-CU, timezone=ART
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=297768800,total=397410304
   [junit4]   2> NOTE: All tests run in this JVM: [TestLucene70DocValuesFormat, TestNativeFSLockFactory, TestNeedsScores, TestSizeBoundedForceMerge, TestFilterIterator, TestMergeRateLimiter, Test2BDocs, TestReaderClosed, TestPerFieldDocValuesFormat, TestIndexWriterDelete, TestNorms, TestTerms, TestApproximationSearchEquivalence, TestAtomicUpdate, TestCustomTermFreq, TestOfflineSorter, TestIntroSelector, TestTermQuery, TestBooleanRewrites, TestQueryRescorer, TestFieldType, TestSpanFirstQuery, TestFilterLeafReader, TestElevationComparator, TestRecyclingIntBlockAllocator, TestGraphTokenStreamFiniteStrings, TestOmitPositions, TestScoreCachingWrappingScorer, TestIndexWriterOnJRECrash, TestLockFactory, TestIndexingSequenceNumbers, TestFloatRangeFieldQueries, TestLevenshteinAutomata, TestDirectPacked, TestWildcard, TestLucene60FieldInfoFormat, TestDoc, TestLucene50StoredFieldsFormat, TestOperations, TestAutomatonQueryUnicode, TestOrdinalMap, TestBooleanQuery, TestTransactionRollback, TestVirtualMethod, TestMultiCollector, TestIndexWriterOnVMError]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterOnVMError -Dtests.seed=4C928641B0C06BCD -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-CU -Dtests.timezone=ART -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | TestIndexWriterOnVMError (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4C928641B0C06BCD]:0)
   [junit4] Completed [358/458 (1!)] on J1 in 7224.25s, 1 test, 2 errors <<< FAILURES!

[...truncated 2979 lines...]
   [junit4] Suite: org.apache.lucene.analysis.opennlp.TestOpenNLPPOSFilterFactory
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestOpenNLPPOSFilterFactory -Dtests.method=testBasic -Dtests.seed=48D3596BCDAE3476 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=th -Dtests.timezone=US/Hawaii -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.20s J2 | TestOpenNLPPOSFilterFactory.testBasic <<<
   [junit4]    > Throwable #1: org.junit.ComparisonFailure: term 0 expected:<[Senntennce]> but was:<[.]>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([48D3596BCDAE3476:E329447E1272B258]:0)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents(BaseTokenStreamTestCase.java:201)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents(BaseTokenStreamTestCase.java:325)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents(BaseTokenStreamTestCase.java:329)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:865)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:727)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:723)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo(BaseTokenStreamTestCase.java:384)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo(BaseTokenStreamTestCase.java:420)
   [junit4]    > 	at org.apache.lucene.analysis.opennlp.TestOpenNLPPOSFilterFactory.testBasic(TestOpenNLPPOSFilterFactory.java:67)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=th, timezone=US/Hawaii
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=212376744,total=253231104
   [junit4]   2> NOTE: All tests run in this JVM: [TestOpenNLPPOSFilterFactory]
   [junit4] Completed [1/5 (1!)] on J2 in 7.40s, 3 tests, 1 failure <<< FAILURES!

[...truncated 6049 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> Dec 19, 2017 3:40:22 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    2) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    3) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[C2908D1F73859111], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersectionBounds(Plane.java:1073)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1214)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addIntersection(XYZBounds.java:262)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoStandardPath$PathSegment.getBounds(GeoStandardPath.java:1147)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoStandardPath.getBounds(GeoStandardPath.java:427)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    4) Thread[id=13, name=SUITE-TestGeo3DPoint-seed#[C2908D1F73859111], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant t

[...truncated too long message...]

      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Dec 19, 2017 7:59:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=32143, name=TEST-TestGenericDistributedQueue.testDistributedQueue-seed#[4C19B6982B75C414], state=TIMED_WAITING, group=TGRP-TestGenericDistributedQueue]
   [junit4]   2> 17474502 INFO  (TEST-TestGenericDistributedQueue.testDistributedQueue-seed#[4C19B6982B75C414]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDistributedQueue
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestGenericDistributedQueue_4C19B6982B75C414-001
   [junit4]   2> Dec 19, 2017 9:59:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=ro-RO, timezone=BET
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=166597768,total=522190848
   [junit4]   2> NOTE: All tests run in this JVM: [XsltUpdateRequestHandlerTest, TestCollectionsAPIViaSolrCloudCluster, TestCharFilters, ExplicitHLLTest, TestInitQParser, StatsComponentTest, TestTrackingShardHandlerFactory, TestTrie, MetricsConfigTest, TestSolrDeletionPolicy2, ConfigSetsAPITest, DistributedSpellCheckComponentTest, TestIntervalFaceting, SharedFSAutoReplicaFailoverTest, TestRequestForwarding, TestExtendedDismaxParser, TestCoreContainer, TestUnifiedSolrHighlighter, IndexBasedSpellCheckerTest, OverseerStatusTest, TestReqParamsAPI, DistribJoinFromCollectionTest, DocValuesNotIndexedTest, TestUseDocValuesAsStored, BaseCdcrDistributedZkTest, TestRandomRequestDistribution, TestBinaryResponseWriter, SuggesterWFSTTest, TestSolrCloudWithSecureImpersonation, ZkStateWriterTest, RequiredFieldsTest, TestDistribDocBasedVersion, QueryElevationComponentTest, ChaosMonkeyNothingIsSafeTest, TestSolrCoreProperties, WordBreakSolrSpellCheckerTest, TestPushWriter, TestLFUCache, BigEndianAscendingWordDeserializerTest, SystemInfoHandlerTest, SuggestComponentContextFilterQueryTest, ConnectionReuseTest, TestCloudPivotFacet, ComputePlanActionTest, TestManagedResource, DeleteNodeTest, TestXmlQParser, TestCorePropertiesReload, TimeRoutedAliasUpdateProcessorTest, TestFilteredDocIdSet, ReplicationFactorTest, TestQuerySenderListener, BasicDistributedZk2Test, TestPolicyCloud, UnloadDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest, TestReload, TestQueryTypes, ResponseHeaderTest, DirectSolrConnectionTest, SolrCoreCheckLockOnStartupTest, CachingDirectoryFactoryTest, BooleanFieldTest, SolrTestCaseJ4Test, TestDFISimilarityFactory, SpatialRPTFieldTypeTest, CopyFieldTest, TestSearcherReuse, SimplePostToolTest, VersionInfoTest, CloudMLTQParserTest, DisMaxRequestHandlerTest, SuggesterTest, MoveReplicaHDFSTest, TestTestInjection, TestCoreBackup, DefaultValueUpdateProcessorTest, ManagedSchemaRoundRobinCloudTest, DistributedSuggestComponentTest, DistributedExpandComponentTest, SolrMetricReporterTest, TestDocBasedVersionConstraints, TestComplexPhraseQParserPlugin, EchoParamsTest, AddBlockUpdateTest, TestScoreJoinQPNoScore, TestHdfsCloudBackupRestore, TestRealTimeGet, TestNumericTerms32, LeaderElectionContextKeyTest, TestDynamicFieldResource, FullSolrCloudDistribCmdsTest, AsyncCallRequestStatusResponseTest, LoggingHandlerTest, TestImplicitCoreProperties, TestSolrCloudSnapshots, TestManagedSynonymFilterFactory, TestSerializedLuceneMatchVersion, TestLRUCache, IndexSchemaTest, TestValueSourceCache, TestMultiValuedNumericRangeQuery, CollectionReloadTest, CustomCollectionTest, SolrGraphiteReporterTest, BlockDirectoryTest, ZkNodePropsTest, CoreAdminHandlerTest, HdfsLockFactoryTest, DeleteStatusTest, SearchRateTriggerTest, SolrCloudExampleTest, AnalyticsQueryTest, TestDistributedMissingSort, TestHashPartitioner, TestLMDirichletSimilarityFactory, TestReRankQParserPlugin, TriggerIntegrationTest, TestRetrieveFieldsOptimizer, HdfsNNFailoverTest, DirectSolrSpellCheckerTest, TestSolrCloudWithHadoopAuthPlugin, TestCloudSchemaless, TestOverriddenPrefixQueryForCustomFieldType, CollectionsAPIDistributedZkTest, SyncSliceTest, OverseerTest, ShardRoutingTest, BasicZkTest, RecoveryZkTest, ClusterStateUpdateTest, TestRandomFaceting, ShardRoutingCustomTest, TestDistributedGrouping, TestFaceting, TestRecovery, ZkControllerTest, TestJoin, TestStressVersions, DistributedTermsComponentTest, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, SpellCheckComponentTest, TestGroupingSearch, SolrCmdDistributorTest, PeerSyncTest, ConvertedLegacyTest, TestSort, TestFiltering, TestFunctionQuery, BasicFunctionalityTest, DirectUpdateHandlerTest, TestIndexSearcher, HighlighterTest, SoftAutoCommitTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, DocValuesTest, TestStressLucene, SpatialFilterTest, TestCSVLoader, PolyFieldTest, SchemaVersionSpecificBehaviorTest, FieldMutatingUpdateProcessorTest, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, TestRemoteStreaming, DistanceFunctionTest, SolrInfoBeanTest, DebugComponentTest, TestSurroundQueryParser, TestQueryUtils, TestWriterPerf, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, TermsComponentTest, XmlUpdateRequestHandlerTest, DocumentBuilderTest, MoreLikeThisHandlerTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, FastVectorHighlighterTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, TestCSVResponseWriter, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, SearchHandlerTest, HighlighterConfigTest, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, TestCursorMarkWithoutUniqueKey, TestTolerantSearch, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, ActionThrottleTest, AddReplicaTest, AliasIntegrationTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, ForceLeaderTest, LeaderInitiatedRecoveryOnShardRestartTest, LegacyCloudClusterPropTest, MigrateRouteKeyTest, MissingSegmentRecoveryTest, MoveReplicaHDFSFailoverTest, RecoveryAfterSoftCommitTest, RemoteQueryErrorTest, SSLMigrationTest, SaslZkACLProviderTest, ShardSplitTest, TestRebalanceLeaders, TestSSLRandomization, TestSegmentSorting, TestSkipOverseerOperations, TestStressInPlaceUpdates, TestTolerantUpdateProcessorRandomCloud, TlogReplayBufferedWhileIndexingTest, AutoAddReplicasIntegrationTest, HttpTriggerListenerTest, NodeAddedTriggerTest, TestClusterStateProvider, TestDistribStateManager, TestGenericDistributedQueue]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGenericDistributedQueue -Dtests.seed=4C19B6982B75C414 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ro-RO -Dtests.timezone=BET -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestGenericDistributedQueue (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4C19B6982B75C414]:0)
   [junit4] Completed [763/763 (9!)] on J2 in 7220.64s, 2 tests, 2 errors <<< FAILURES!

[...truncated 48089 lines...]

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 106 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/106/

10 tests failed.
FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([FF3DEC673A40CD69]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=1904, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[83BCDB0E2B8B46F4]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    2) Thread[id=1902, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=1928, name=zkCallback-445-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=2526, name=zkCallback-445-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=3150, name=zkCallback-445-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=1905, name=zkConnectionManagerCallback-446-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=2579, name=zkCallback-445-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=1903, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[83BCDB0E2B8B46F4]-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)         at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170)    9) Thread[id=2525, name=zkCallback-445-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=1904, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[83BCDB0E2B8B46F4]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   2) Thread[id=1902, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=1928, name=zkCallback-445-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=2526, name=zkCallback-445-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=3150, name=zkCallback-445-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=1905, name=zkConnectionManagerCallback-446-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=2579, name=zkCallback-445-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=1903, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[83BCDB0E2B8B46F4]-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
        at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170)
   9) Thread[id=2525, name=zkCallback-445-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1928, name=zkCallback-445-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=2526, name=zkCallback-445-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=3150, name=zkCallback-445-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=1905, name=zkConnectionManagerCallback-446-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=2579, name=zkCallback-445-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=1903, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[83BCDB0E2B8B46F4]-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    7) Thread[id=2525, name=zkCallback-445-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1928, name=zkCallback-445-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=2526, name=zkCallback-445-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=3150, name=zkCallback-445-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=1905, name=zkConnectionManagerCallback-446-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=2579, name=zkCallback-445-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=1903, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[83BCDB0E2B8B46F4]-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   7) Thread[id=2525, name=zkCallback-445-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:351)  at org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3031)  at org.apache.solr.core.SolrCore.close(SolrCore.java:1604)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1000)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:640)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:351)
	at org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3031)
	at org.apache.solr.core.SolrCore.close(SolrCore.java:1604)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1000)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039)
	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:640)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:297)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit

Error Message:
refusing to delete any files: this IndexWriter hit an unrecoverable exception

Stack Trace:
org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4:B00EF3CA263C9C43]:0)
	at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:345)
	at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:697)
	at org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:594)
	at org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:533)
	at org.apache.lucene.index.IndexWriter.checkpoint(IndexWriter.java:2649)
	at org.apache.lucene.index.IndexWriter.publishFlushedSegment(IndexWriter.java:2717)
	at org.apache.lucene.index.DocumentsWriterFlushQueue$FlushTicket.publishFlushedSegment(DocumentsWriterFlushQueue.java:201)
	at org.apache.lucene.index.DocumentsWriterFlushQueue$FlushTicket.finishFlush(DocumentsWriterFlushQueue.java:215)
	at org.apache.lucene.index.DocumentsWriterFlushQueue$SegmentFlushTicket.publish(DocumentsWriterFlushQueue.java:252)
	at org.apache.lucene.index.DocumentsWriterFlushQueue.innerPurge(DocumentsWriterFlushQueue.java:116)
	at org.apache.lucene.index.DocumentsWriterFlushQueue.tryPurge(DocumentsWriterFlushQueue.java:153)
	at org.apache.lucene.index.DocumentsWriter.purgeBuffer(DocumentsWriter.java:202)
	at org.apache.lucene.index.IndexWriter.purge(IndexWriter.java:5051)
	at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:5064)
	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:601)
	at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:415)
	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:473)
	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1532)
	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1508)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:141)
	at org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit(TestDocTermOrds.java:167)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:456)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:129)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:113)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:292)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:372)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:112)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:78)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:208)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4588)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4083)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testRandom

Error Message:
Captured an uncaught exception in thread: Thread[id=12716, name=Lucene Merge Thread #191, state=RUNNABLE, group=TGRP-TestDocTermOrds]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=12716, name=Lucene Merge Thread #191, state=RUNNABLE, group=TGRP-TestDocTermOrds]
Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot complete merge
	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4]:0)
	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:703)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:683)
Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot complete merge
	at org.apache.lucene.index.IndexWriter.commitMerge(IndexWriter.java:3890)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4599)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4083)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:456)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:129)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:113)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:292)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:372)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:112)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:78)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:208)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4588)
	... 3 more


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:35123/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35123/collection1
	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4:BE8E4D485772B0C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1583)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:212)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 53 more


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10003_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10003_solr should have been deleted
	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4:9B06530225BE8B1B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4:DEF4C587E44DE0BB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1178)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([FF3DEC673A40CD69]:0)




Build Log:
[...truncated 10934 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> Dec 17, 2017 1:06:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=13, name=SUITE-TestGeo3DPoint-seed#[FF3DEC673A40CD69], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    2) Thread[id=14, name=TEST-TestGeo3DPoint.testRandomBig-seed#[FF3DEC673A40CD69], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addPlane(XYZBounds.java:161)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoStandardPath$PathSegment.getBounds(GeoStandardPath.java:1140)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoStandardPath.getBounds(GeoStandardPath.java:427)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=FF3DEC673A40CD69 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-SG -Dtests.timezone=Europe/Kiev -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7200s J1 | TestGeo3DPoint.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FF3DEC673A40CD69]:0)
   [junit4]   2> Dec 17, 2017 1:06:43 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Asserting)}, docValues:{id=DocValuesFormat(name=Lucene70), point=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1098, maxMBSortInHeap=5.48360341697566, sim=RandomSimilarity(queryNorm=true): {}, locale=en-SG, timezone=Europe/Kiev
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=41252704,total=486014976
   [junit4]   2> NOTE: All tests run in this JVM: [GeoCircleTest, TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=FF3DEC673A40CD69 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-SG -Dtests.timezone=Europe/Kiev -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FF3DEC673A40CD69]:0)
   [junit4] Completed [15/16 (1!)] on J1 in 7215.67s, 2 tests, 2 errors <<< FAILURES!

[...truncated 1987 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> 146196 INFO  (SUITE-TestTriggerIntegration-seed#[83BCDB0E2B8B46F4]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_83BCDB0E2B8B46F4-001/init-core-data-001
   [junit4]   2> 146214 WARN  (SUITE-TestTriggerIntegration-seed#[83BCDB0E2B8B46F4]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 146214 INFO  (SUITE-TestTriggerIntegration-seed#[83BCDB0E2B8B46F4]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 146215 INFO  (SUITE-TestTriggerIntegration-seed#[83BCDB0E2B8B46F4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 146378 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 146525 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 146525 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 146525 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 146525 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 146690 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 146690 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 146837 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration
   [junit4]   2> 146837 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 146838 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 146838 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 146838 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 146836 DEBUG (ScheduledTrigger-120-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 146852 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 146853 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 146853 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 146853 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 146853 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 146853 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1
   [junit4]   2> 146853 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 146854 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 146873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 146873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 146873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 146873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 146873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10000_solr
   [junit4]   2> 146889 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 146889 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 146889 DEBUG (ScheduledTrigger-124-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 146889 DEBUG (ScheduledTrigger-124-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr
   [junit4]   2> 146925 DEBUG (ScheduledTrigger-124-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 146962 DEBUG (ScheduledTrigger-124-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 146999 DEBUG (ScheduledTrigger-124-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 146999 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS
   [junit4]   2> 147035 DEBUG (ScheduledTrigger-124-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 147071 DEBUG (ScheduledTrigger-124-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 147111 DEBUG (ScheduledTrigger-124-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 147144 DEBUG (ScheduledTrigger-124-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 147197 DEBUG (ScheduledTrigger-124-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 147202 DEBUG (simCloudManagerPool-119-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 147203 DEBUG (simCloudManagerPool-119-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 147218 DEBUG (ScheduledTrigger-124-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 147218 DEBUG (simCloudManagerPool-119-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 147220 DEBUG (simCloudManagerPool-119-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 147220 DEBUG (simCloudManagerPool-119-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true}
   [junit4]   2> 147220 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 147255 DEBUG (ScheduledTrigger-124-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 147220 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init
   [junit4]   2> 147256 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 147256 DEBUG (ScheduledTrigger-124-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 147256 DEBUG (ScheduledTrigger-124-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 147255 DEBUG (simCloudManagerPool-119-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 147271 DEBUG (simCloudManagerPool-119-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 147271 DEBUG (simCloudManagerPool-119-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 147271 DEBUG (simCloudManagerPool-119-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 147271 DEBUG (simCloudManagerPool-119-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 147328 DEBUG (simCloudManagerPool-119-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true}
   [junit4]   2> 147328 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 147328 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1
   [junit4]   2> 147329 DEBUG (ScheduledTrigger-124-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 147329 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init
   [junit4]   2> 147330 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 147330 DEBUG (ScheduledTrigger-124-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3
   [junit4]   2> 147364 DEBUG (ScheduledTrigger-124-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 147364 DEBUG (ScheduledTrigger-124-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3
   [junit4]   2> 147364 DEBUG (ScheduledTrigger-124-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 147365 DEBUG (ScheduledTrigger-124-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 147365 DEBUG (ScheduledTrigger-124-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 13622216875569243
   [junit4]   2> 147400 DEBUG (ScheduledTrigger-124-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 147400 DEBUG (ScheduledTrigger-124-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3
   [junit4]   2> 147401 DEBUG (ScheduledTrigger-124-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 147401 DEBUG (ScheduledTrigger-124-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 147401 DEBUG (ScheduledTrigger-124-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [13622216875569243], now=13622218657196343
   [junit4]   2> 147401 DEBUG (ScheduledTrigger-124-thread-3) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"30655585ebf05bT46wmnqax5ybd2hp2o1jk1if3",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":13622216875569243,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[13622216875569243],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 147437 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeMarkersRegistration
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeMarkersRegistration -Dtests.seed=83BCDB0E2B8B46F4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=Asia/Amman -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 1.08s J1 | TestTriggerIntegration.testNodeMarkersRegistration <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10003_solr should have been deleted
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4:9B06530225BE8B1B]:0)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 147491 DEBUG (AutoscalingActionExecutor-125-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"30655585ebf05bT46wmnqax5ybd2hp2o1jk1if3",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":13622216875569243,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[13622216875569243],
   [junit4]   2>     "_enqueue_time_":13622218685625893,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 147548 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
   [junit4]   2> 147548 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 147548 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 147548 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 147549 WARN  (AutoscalingActionExecutor-125-thread-1) [    ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system
   [junit4]   2> java.io.IOException: java.lang.InterruptedException
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:430)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:122)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:744)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:714)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:340)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404)
   [junit4]   2> 	at java.util.concurrent.FutureTask.get(FutureTask.java:191)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:428)
   [junit4]   2> 	... 10 more
   [junit4]   2> 147549 DEBUG (AutoscalingActionExecutor-125-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 2933 ms for event id=30655585ebf05bT46wmnqax5ybd2hp2o1jk1if3
   [junit4]   2> 147549 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[83BCDB0E2B8B46F4]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 147564 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 147565 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 147565 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10004_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 147565 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 147565 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 147579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 147579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 147580 DEBUG (ScheduledTrigger-126-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 147600 DEBUG (ScheduledTrigger-126-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 147640 DEBUG (simCloudManagerPool-119-thread-5) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 147640 DEBUG (ScheduledTrigger-126-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 147660 DEBUG (ScheduledTrigger-126-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 147673 DEBUG (simCloudManagerPool-119-thread-5) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 147689 DEBUG (ScheduledTrigger-126-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 147710 DEBUG (ScheduledTrigger-126-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
   [junit4]   2> 147710 DEBUG (simCloudManagerPool-119-thread-5) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10004_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 147710 DEBUG (simCloudManagerPool-119-thread-5) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10004_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 147710 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 147711 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 147711 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 147711 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 147711 DEBUG (ScheduledTrigger-126-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 4
   [junit4]   2> 147730 DEBUG (ScheduledTrigger-126-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 147730 DEBUG (ScheduledTrigger-126-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 147731 DEBUG (ScheduledTrigger-126-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 3
   [junit4]   2> 147731 DEBUG (ScheduledTrigger-126-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 147766 DEBUG (ScheduledTrigger-126-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 147767 DEBUG (ScheduledTrigger-126-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 3
   [junit4]   2> 147767 DEBUG (simCloudManagerPool-119-thread-6) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 147767 DEBUG (simCloudManagerPool-119-thread-6) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
   [junit4]   2> 147768 DEBUG (simCloudManagerPool-119-thread-6) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 147768 DEBU

[...truncated too long message...]

run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> Signal Dispatcher:
   [junit4]   2> 
   [junit4]   2> org.apache.hadoop.hdfs.PeerCache@7ad8fb0f:
   [junit4]   2>     java.lang.Thread.sleep(Native Method)
   [junit4]   2>     org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255)
   [junit4]   2>     org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
   [junit4]   2>     org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
   [junit4]   2>     java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner:
   [junit4]   2>     java.lang.Object.wait(Native Method)
   [junit4]   2>     java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
   [junit4]   2>     java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
   [junit4]   2>     org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3063)
   [junit4]   2>     java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> Finalizer:
   [junit4]   2>     java.lang.Object.wait(Native Method)
   [junit4]   2>     java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
   [junit4]   2>     java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
   [junit4]   2>     java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
   [junit4]   2> 
   [junit4]   2> Reference Handler:
   [junit4]   2>     java.lang.Object.wait(Native Method)
   [junit4]   2>     java.lang.Object.wait(Object.java:502)
   [junit4]   2>     java.lang.ref.Reference.tryHandlePending(Reference.java:191)
   [junit4]   2>     java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)
   [junit4]   2> 
   [junit4]   2> IPC Parameter Sending Thread #15:
   [junit4]   2>     [...sun.*]
   [junit4]   2>     java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>     java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2>     java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2>     java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2>     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]   2>     java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>     java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> JUnit4-serializer-daemon:
   [junit4]   2>     java.lang.Thread.sleep(Native Method)
   [junit4]   2>     com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> 
   [junit4]   2> 6871882 INFO  (SUITE-HdfsBasicDistributedZkTest-seed#[83BCDB0E2B8B46F4]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {regex_dup_A_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), regex_dup_B_s=PostingsFormat(name=Direct), SubjectTerms_mfacet=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=LuceneFixedGap), genre_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), author_t=PostingsFormat(name=LuceneFixedGap), series_t=PostingsFormat(name=LuceneFixedGap), rnd_b=BlockTreeOrds(blocksize=128), oddField_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), a_t=PostingsFormat(name=Direct), cat=PostingsFormat(name=Direct), foo_b=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), name=BlockTreeOrds(blocksize=128), inStock=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=BlockTreeOrds(blocksize=128), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{regex_dup_A_s=DocValuesFormat(name=Lucene70), other_tl1=DocValuesFormat(name=Memory), regex_dup_B_s=DocValuesFormat(name=Memory), range_facet_l_dv=DocValuesFormat(name=Direct), SubjectTerms_mfacet=DocValuesFormat(name=Memory), n_l1=DocValuesFormat(name=Memory), multiDefault=DocValuesFormat(name=Asserting), intDefault=DocValuesFormat(name=Memory), genre_s=DocValuesFormat(name=Lucene70), n_td1=DocValuesFormat(name=Asserting), n_d1=DocValuesFormat(name=Memory), range_facet_l=DocValuesFormat(name=Memory), n_f1=DocValuesFormat(name=Asserting), series_t=DocValuesFormat(name=Asserting), n_tl1=DocValuesFormat(name=Asserting), oddField_s=DocValuesFormat(name=Lucene70), n_tf1=DocValuesFormat(name=Memory), price=DocValuesFormat(name=Lucene70), a_t=DocValuesFormat(name=Memory), cat=DocValuesFormat(name=Memory), sequence_i=DocValuesFormat(name=Lucene70), inStock=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Direct), text=DocValuesFormat(name=Lucene70), intDvoDefault=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Memory), foo_i=DocValuesFormat(name=Asserting), val_i=DocValuesFormat(name=Direct), n_dt1=DocValuesFormat(name=Asserting), author_t=DocValuesFormat(name=Asserting), a_i1=DocValuesFormat(name=Memory), n_ti1=DocValuesFormat(name=Direct), rnd_b=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Memory), n_tdt1=DocValuesFormat(name=Memory), foo_b=DocValuesFormat(name=Lucene70), name=DocValuesFormat(name=Direct), id_i1=DocValuesFormat(name=Asserting), foo_d=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Memory), foo_f=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=612, maxMBSortInHeap=6.41710984782834, sim=RandomSimilarity(queryNorm=false): {}, locale=id, timezone=MST
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=7,free=250442120,total=525336576
   [junit4]   2> NOTE: All tests run in this JVM: [TestTrieFacet, DistributedQueryComponentOptimizationTest, TestFieldResource, TestNoOpRegenerator, TestConfigSets, TestTriggerIntegration, HLLSerializationTest, MoreLikeThisHandlerTest, TestDeleteCollectionOnDownNodes, TestLegacyTerms, CdcrUpdateLogTest, RollingRestartTest, TestLegacyFieldCache, TestFieldSortValues, TestSolrFieldCacheBean, TestChildDocTransformer, SuggesterTSTTest, TermVectorComponentDistributedTest, ShardSplitTest, BadCopyFieldTest, ZkFailoverTest, TestCloudManagedSchema, TestLegacyField, SynonymTokenizerTest, HighlighterConfigTest, TestConfigReload, SpatialHeatmapFacetsTest, TestDistributedStatsComponentCardinality, TestSurroundQueryParser, TestPayloadScoreQParserPlugin, HdfsBasicDistributedZkTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsBasicDistributedZkTest -Dtests.seed=83BCDB0E2B8B46F4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=id -Dtests.timezone=MST -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | HdfsBasicDistributedZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory]
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
   [junit4]    > 	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:351)
   [junit4]    > 	at org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3031)
   [junit4]    > 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1604)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1000)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:640)
   [junit4]    > 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([83BCDB0E2B8B46F4]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:297)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [666/763 (4!)] on J1 in 3412.06s, 1 test, 1 failure <<< FAILURES!

[...truncated 8870 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:843: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:787: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid17411.hprof

Total time: 820 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 105 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/105/

11 tests failed.
FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([2D58666BA8CA1695]:0)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10003_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10003_solr should have been deleted
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA:9A9CB958C6BA1355]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA:DF6E2FDD074978F5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1178)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync

Error Message:
Could not load collection from ZK: source_collection

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: source_collection
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA:908111BD64FF35FC]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1108)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
	at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:137)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
	at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.waitForRecoveriesToFinish(BaseCdcrDistributedZkTest.java:522)
	at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.restartServer(BaseCdcrDistributedZkTest.java:563)
	at org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync(CdcrReplicationHandlerTest.java:205)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/source_collection/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:339)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1120)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1106)
	... 49 more


FAILED:  org.apache.solr.core.TestLazyCores.testNoCommit

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA:5D46908503A8BD1F]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:901)
	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:847)
	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:829)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int><lst name="params"><str name="q">*:*</str></lst></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=*:*
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:894)
	... 41 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=24574, name=searcherExecutor-4953-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   1) Thread[id=24574, name=searcherExecutor-4953-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=24574, name=searcherExecutor-4953-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24574, name=searcherExecutor-4953-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA]:0)


FAILED:  org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory

Error Message:
expected:<8> but was:<10>

Stack Trace:
java.lang.AssertionError: expected:<8> but was:<10>
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA:EFDA95A972C721BD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:259)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([2D58666BA8CA1695]:0)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA:7B6BA2FBF4FA9330]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:279)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/30)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{"core_node2":{           "core":"solrj_collection3_shard1_replica_n1",           "base_url":"http://127.0.0.1:44299",           "node_name":"127.0.0.1:44299_",           "state":"down",           "type":"NRT"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{"core_node4":{           "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node4/data/",           "base_url":"http://127.0.0.1:44299",           "node_name":"127.0.0.1:44299_",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node4/data/tlog",           "core":"solrj_collection3_shard2_replica_n3",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{"core_node7":{           "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node7/data/",           "base_url":"http://127.0.0.1:36554",           "node_name":"127.0.0.1:36554_",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node7/data/tlog",           "core":"solrj_collection3_shard3_replica_n5",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{"core_node9":{           "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node9/data/",           "base_url":"http://127.0.0.1:44299",           "node_name":"127.0.0.1:44299_",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",           "core":"solrj_collection3_shard4_replica_n6",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{"core_node10":{           "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node10/data/",           "base_url":"http://127.0.0.1:36554",           "node_name":"127.0.0.1:36554_",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",           "core":"solrj_collection3_shard5_replica_n8",           "shared_storage":"true",           "state":"active",           "leader":"true"}}}},   "router":{     "field":"myOwnField",     "name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"true",   "nrtReplicas":"1",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:33534_, 127.0.0.1:35054_, 127.0.0.1:36554_, 127.0.0.1:39799_, 127.0.0.1:44299_]

Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/30)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{"core_node2":{
          "core":"solrj_collection3_shard1_replica_n1",
          "base_url":"http://127.0.0.1:44299",
          "node_name":"127.0.0.1:44299_",
          "state":"down",
          "type":"NRT"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{"core_node4":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node4/data/",
          "base_url":"http://127.0.0.1:44299",
          "node_name":"127.0.0.1:44299_",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node4/data/tlog",
          "core":"solrj_collection3_shard2_replica_n3",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{"core_node7":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node7/data/",
          "base_url":"http://127.0.0.1:36554",
          "node_name":"127.0.0.1:36554_",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node7/data/tlog",
          "core":"solrj_collection3_shard3_replica_n5",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{"core_node9":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node9/data/",
          "base_url":"http://127.0.0.1:44299",
          "node_name":"127.0.0.1:44299_",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",
          "core":"solrj_collection3_shard4_replica_n6",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{"core_node10":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node10/data/",
          "base_url":"http://127.0.0.1:36554",
          "node_name":"127.0.0.1:36554_",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",
          "core":"solrj_collection3_shard5_replica_n8",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}}},
  "router":{
    "field":"myOwnField",
    "name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"true",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:33534_, 127.0.0.1:35054_, 127.0.0.1:36554_, 127.0.0.1:39799_, 127.0.0.1:44299_]
	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA:A720E8E6673B342]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:414)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:256)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:144)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 10748 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> ??? 15, 2017 4:28:58 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[2D58666BA8CA1695], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    2) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomMedium-seed#[2D58666BA8CA1695], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1251)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addPlane(XYZBounds.java:161)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoConvexPolygon.getBounds(GeoConvexPolygon.java:484)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseCompositeShape.getBounds(GeoBaseCompositeShape.java:131)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium(TestGeo3DPoint.java:475)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomMedium -Dtests.seed=2D58666BA8CA1695 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-EG -Dtests.timezone=America/Hermosillo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    379s J1 | TestGeo3DPoint.testRandomMedium <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2D58666BA8CA1695]:0)
   [junit4]   2> ??? 15, 2017 4:28:58 ? com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> ??? 15, 2017 4:29:18 ? com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint: 
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomMedium-seed#[2D58666BA8CA1695], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.findIntersectionBounds(Plane.java:1105)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1214)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addIntersection(XYZBounds.java:262)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoConvexPolygon.getBounds(GeoConvexPolygon.java:486)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseCompositeShape.getBounds(GeoBaseCompositeShape.java:131)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium(TestGeo3DPoint.java:475)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> ??? 15, 2017 4:29:18 ? com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomMedium-seed#[2D58666BA8CA1695], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> ??? 15, 2017 4:29:21 ? com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomMedium-seed#[2D58666BA8CA1695], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1583)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.XYZBounds.addPlane(XYZBounds.java:161)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoConvexPolygon.getBounds(GeoConvexPolygon.java:484)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseCompositeShape.getBounds(GeoBaseCompositeShape.java:131)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomMedium(TestGeo3DPoint.java:475)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Asserting)}, docValues:{id=DocValuesFormat(name=Lucene70), point=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=759, maxMBSortInHeap=6.682149951690519, sim=RandomSimilarity(queryNorm=false): {}, locale=ar-EG, timezone=America/Hermosillo
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=167581752,total=398458880
   [junit4]   2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=2D58666BA8CA1695 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-EG -Dtests.timezone=America/Hermosillo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2D58666BA8CA1695]:0)
   [junit4] Completed [15/16 (1!)] on J1 in 7225.44s, 12 tests, 2 errors <<< FAILURES!

[...truncated 2014 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_82263154C88FDEBA-001/init-core-data-001
   [junit4]   2> 868281 WARN  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
   [junit4]   2> 868281 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 868283 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 868283 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 868378 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=clusterstate.json
   [junit4]   2> 868378 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=clusterprops.json
   [junit4]   2> 868378 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling.json
   [junit4]   2> 868378 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=live_nodes
   [junit4]   2> 868378 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=roles.json
   [junit4]   2> 868379 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868379 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=events
   [junit4]   2> 868379 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868379 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868379 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868379 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=nodeLost
   [junit4]   2> 868379 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868379 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=nodeAdded
   [junit4]   2> 868431 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=live_nodes
   [junit4]   2> 868431 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@4a1d8cf3, currentName=127.0.0.1:10000_solr
   [junit4]   2> 868431 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=live_nodes
   [junit4]   2> 868431 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@4a1d8cf3, currentName=127.0.0.1:10000_solr
   [junit4]   2> 868431 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling.json
   [junit4]   2> 868432 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=live_nodes
   [junit4]   2> 868432 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@4a1d8cf3, currentName=127.0.0.1:10001_solr
   [junit4]   2> 868432 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=live_nodes
   [junit4]   2> 868432 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@4a1d8cf3, currentName=127.0.0.1:10001_solr
   [junit4]   2> 868432 INFO  (SUITE-TestTriggerIntegration-seed#[82263154C88FDEBA]-worker) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling.json
   [junit4]   2> 868476 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling.json
   [junit4]   2> 868476 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 868476 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling.json
   [junit4]   2> 868476 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling.json
   [junit4]   2> 868476 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 868477 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868477 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868477 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 868477 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 868477 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 868479 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868479 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=events
   [junit4]   2> 868479 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@1b009d9, currentName=.auto_add_replicas
   [junit4]   2> 868479 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868479 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=events
   [junit4]   2> 868479 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@1b009d9, currentName=.auto_add_replicas
   [junit4]   2> 868479 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868479 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=nodeLost
   [junit4]   2> 868484 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 868484 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868484 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=nodeAdded
   [junit4]   2> 868484 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 868507 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868507 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=events
   [junit4]   2> 868507 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@1b009d9, currentName=.auto_add_replicas
   [junit4]   2> 868520 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration
   [junit4]   2> 868520 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling.json
   [junit4]   2> 868520 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling.json
   [junit4]   2> 868520 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 868520 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868520 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=events
   [junit4]   2> 868520 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868520 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=events
   [junit4]   2> 868520 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=events
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@1b009d9, currentName=.auto_add_replicas
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=events
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@1b009d9, currentName=.auto_add_replicas
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=nodeLost
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=nodeLost
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=nodeAdded
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868521 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=nodeAdded
   [junit4]   2> 868522 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 868522 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 868543 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@31028c04, currentName=.auto_add_replicas
   [junit4]   2> 868544 DEBUG (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@31028c04, currentName=.auto_add_replicas
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@31028c04, currentName=.auto_add_replicas
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=autoscaling
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@3f319405, currentName=triggerState
   [junit4]   2> 868544 INFO  (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@31028c04, currentName=.auto_add_replicas
   [junit4]   2> 868593 DEBUG (ScheduledTrigger-290-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 868630 WARN  (ScheduledTrigger-290-thread-3) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger ran but was already closed
   [junit4]   2> 868630 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 868631 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=live_nodes
   [junit4]   2> 868631 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@4a1d8cf3, currentName=127.0.0.1:10002_solr
   [junit4]   2> 868631 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@20b657cb, currentName=live_nodes
   [junit4]   2> 868631 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager$Node@4a1d8cf3, currentName=127.0.0.1:10002_solr
   [junit4]   2> 868631 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[82263154C88FDEBA]) [    ] o.a.s.c.a.s.SimDistribStateManager  - parentNode=org.apache.solr.cloud.autoscali

[...truncated too long message...]

-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node4/data/",
   [junit4]    >           "base_url":"http://127.0.0.1:44299",
   [junit4]    >           "node_name":"127.0.0.1:44299_",
   [junit4]    >           "type":"NRT",
   [junit4]    >           "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node4/data/tlog",
   [junit4]    >           "core":"solrj_collection3_shard2_replica_n3",
   [junit4]    >           "shared_storage":"true",
   [junit4]    >           "state":"active",
   [junit4]    >           "leader":"true"}}},
   [junit4]    >     "shard3":{
   [junit4]    >       "range":"e6660000-1998ffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{"core_node7":{
   [junit4]    >           "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node7/data/",
   [junit4]    >           "base_url":"http://127.0.0.1:36554",
   [junit4]    >           "node_name":"127.0.0.1:36554_",
   [junit4]    >           "type":"NRT",
   [junit4]    >           "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node7/data/tlog",
   [junit4]    >           "core":"solrj_collection3_shard3_replica_n5",
   [junit4]    >           "shared_storage":"true",
   [junit4]    >           "state":"active",
   [junit4]    >           "leader":"true"}}},
   [junit4]    >     "shard4":{
   [junit4]    >       "range":"19990000-4ccbffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{"core_node9":{
   [junit4]    >           "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node9/data/",
   [junit4]    >           "base_url":"http://127.0.0.1:44299",
   [junit4]    >           "node_name":"127.0.0.1:44299_",
   [junit4]    >           "type":"NRT",
   [junit4]    >           "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",
   [junit4]    >           "core":"solrj_collection3_shard4_replica_n6",
   [junit4]    >           "shared_storage":"true",
   [junit4]    >           "state":"active",
   [junit4]    >           "leader":"true"}}},
   [junit4]    >     "shard5":{
   [junit4]    >       "range":"4ccc0000-7fffffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{"core_node10":{
   [junit4]    >           "dataDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node10/data/",
   [junit4]    >           "base_url":"http://127.0.0.1:36554",
   [junit4]    >           "node_name":"127.0.0.1:36554_",
   [junit4]    >           "type":"NRT",
   [junit4]    >           "ulogDir":"hdfs://lucene2-us-west.apache.org:32813/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",
   [junit4]    >           "core":"solrj_collection3_shard5_replica_n8",
   [junit4]    >           "shared_storage":"true",
   [junit4]    >           "state":"active",
   [junit4]    >           "leader":"true"}}}},
   [junit4]    >   "router":{
   [junit4]    >     "field":"myOwnField",
   [junit4]    >     "name":"compositeId"},
   [junit4]    >   "maxShardsPerNode":"1",
   [junit4]    >   "autoAddReplicas":"true",
   [junit4]    >   "nrtReplicas":"1",
   [junit4]    >   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:33534_, 127.0.0.1:35054_, 127.0.0.1:36554_, 127.0.0.1:39799_, 127.0.0.1:44299_]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([82263154C88FDEBA:A720E8E6673B342]:0)
   [junit4]    > 	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:414)
   [junit4]    > 	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:256)
   [junit4]    > 	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:144)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 5417752 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[82263154C88FDEBA]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 5417808 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[82263154C88FDEBA]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 5417910 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:32813) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 5417910 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:32813) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-2088124311-127.0.0.1-1513341517375 (Datanode Uuid 072f7224-0230-4319-a8b1-1d7502ce29b1) service to lucene2-us-west.apache.org/127.0.0.1:32813
   [junit4]   2> 5417912 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[82263154C88FDEBA]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 5417956 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[82263154C88FDEBA]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 5418057 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:32813) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 5418057 WARN  (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:32813) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-2088124311-127.0.0.1-1513341517375 (Datanode Uuid 9fa6156c-064c-49df-ac75-e1260c8370d0) service to lucene2-us-west.apache.org/127.0.0.1:32813
   [junit4]   2> 5418088 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[82263154C88FDEBA]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:0
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_82263154C88FDEBA-001
   [junit4]   2> Dec 15, 2017 1:37:03 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=7, maxDocsPerChunk=988, blockSize=3), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=7, blockSize=3)), sim=RandomSimilarity(queryNorm=true): {}, locale=mk-MK, timezone=America/Monterrey
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=111666544,total=524812288
   [junit4]   2> NOTE: All tests run in this JVM: [TestNumericTokenStream, CollectionsAPIAsyncDistributedZkTest, TestBulkSchemaAPI, WordBreakSolrSpellCheckerTest, TestCloudDeleteByQuery, TestCloudSchemaless, TestStandardQParsers, DistributedExpandComponentTest, SSLMigrationTest, TestSystemIdResolver, SuggestComponentContextFilterQueryTest, RecoveryZkTest, StressHdfsTest, DistributedFacetPivotLongTailTest, DeleteNodeTest, SolrGraphiteReporterTest, TestMiniSolrCloudClusterSSL, TestDistributedGrouping, DistribJoinFromCollectionTest, XsltUpdateRequestHandlerTest, DisMaxRequestHandlerTest, TestTolerantSearch, SolrSlf4jReporterTest, BooleanFieldTest, HighlighterTest, TestReqParamsAPI, TestManagedSynonymFilterFactory, AutoscalingHistoryHandlerTest, HttpTriggerListenerTest, TestUnifiedSolrHighlighter, TestGroupingSearch, SharedFSAutoReplicaFailoverTest]
   [junit4] Completed [669/763 (6!)] on J2 in 3516.11s, 1 test, 1 failure <<< FAILURES!

[...truncated 47677 lines...]

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 104 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/104/

8 tests failed.
FAILED:  org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([EFC0B33DA0225E8E]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([EFC0B33DA0225E8E]:0)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node4:{"core":"c8n_1x3_lf_shard1_replica_n2","base_url":"http://127.0.0.1:32811","node_name":"127.0.0.1:32811_","state":"active","type":"NRT","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/16)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"c8n_1x3_lf_shard1_replica_n2",           "base_url":"http://127.0.0.1:32811",           "node_name":"127.0.0.1:32811_",           "state":"active",           "type":"NRT",           "leader":"true"},         "core_node5":{           "core":"c8n_1x3_lf_shard1_replica_n1",           "base_url":"http://127.0.0.1:36170",           "node_name":"127.0.0.1:36170_",           "state":"down",           "type":"NRT"},         "core_node6":{           "state":"down",           "base_url":"http://127.0.0.1:45213",           "core":"c8n_1x3_lf_shard1_replica_n3",           "node_name":"127.0.0.1:45213_",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"3",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node4:{"core":"c8n_1x3_lf_shard1_replica_n2","base_url":"http://127.0.0.1:32811","node_name":"127.0.0.1:32811_","state":"active","type":"NRT","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/16)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"c8n_1x3_lf_shard1_replica_n2",
          "base_url":"http://127.0.0.1:32811",
          "node_name":"127.0.0.1:32811_",
          "state":"active",
          "type":"NRT",
          "leader":"true"},
        "core_node5":{
          "core":"c8n_1x3_lf_shard1_replica_n1",
          "base_url":"http://127.0.0.1:36170",
          "node_name":"127.0.0.1:36170_",
          "state":"down",
          "type":"NRT"},
        "core_node6":{
          "state":"down",
          "base_url":"http://127.0.0.1:45213",
          "core":"c8n_1x3_lf_shard1_replica_n3",
          "node_name":"127.0.0.1:45213_",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([1E7F9EF5B5F4982:89B3C635F5A3247A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:169)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
KeeperErrorCode = Session expired for /overseer_elect/leader

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
	at __randomizedtesting.SeedInfo.seed([1E7F9EF5B5F4982:89B3C635F5A3247A]:0)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:339)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
	at org.apache.solr.cloud.OverseerTaskProcessor.getLeaderId(OverseerTaskProcessor.java:374)
	at org.apache.solr.cloud.OverseerTaskProcessor.getLeaderNode(OverseerTaskProcessor.java:365)
	at org.apache.solr.cloud.RollingRestartTest.waitUntilOverseerDesignateIsLeader(RollingRestartTest.java:137)
	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:104)
	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:52)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCreateNodeSet

Error Message:
Could not find collection : nodeset_collection

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : nodeset_collection
	at __randomizedtesting.SeedInfo.seed([1E7F9EF5B5F4982:5A442E0F32DE6D0A]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:111)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCreateNodeSet(CollectionsAPIDistributedZkTest.java:404)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([1E7F9EF5B5F4982:F8AA6A40672A0408]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:279)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/32)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{"core_node4":{           "dataDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node4/data/",           "base_url":"http://127.0.0.1:39323/_mhe/ly",           "node_name":"127.0.0.1:39323__mhe%2Fly",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node4/data/tlog",           "core":"solrj_collection3_shard1_replica_n1",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{"core_node8":{           "dataDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node8/data/",           "base_url":"http://127.0.0.1:41995/_mhe/ly",           "node_name":"127.0.0.1:41995__mhe%2Fly",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node8/data/tlog",           "core":"solrj_collection3_shard2_replica_n2",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{"core_node6":{           "dataDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node6/data/",           "base_url":"http://127.0.0.1:41995/_mhe/ly",           "node_name":"127.0.0.1:41995__mhe%2Fly",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node6/data/tlog",           "core":"solrj_collection3_shard3_replica_n3",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{"core_node9":{           "core":"solrj_collection3_shard4_replica_n5",           "base_url":"http://127.0.0.1:41995/_mhe/ly",           "node_name":"127.0.0.1:41995__mhe%2Fly",           "state":"down",           "type":"NRT"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{"core_node10":{           "dataDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node10/data/",           "base_url":"http://127.0.0.1:39323/_mhe/ly",           "node_name":"127.0.0.1:39323__mhe%2Fly",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",           "core":"solrj_collection3_shard5_replica_n7",           "shared_storage":"true",           "state":"active",           "leader":"true"}}}},   "router":{     "field":"myOwnField",     "name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"true",   "nrtReplicas":"1",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:37530__mhe%2Fly, 127.0.0.1:39323__mhe%2Fly, 127.0.0.1:41313__mhe%2Fly, 127.0.0.1:41995__mhe%2Fly, 127.0.0.1:46306__mhe%2Fly]

Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/32)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{"core_node4":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node4/data/",
          "base_url":"http://127.0.0.1:39323/_mhe/ly",
          "node_name":"127.0.0.1:39323__mhe%2Fly",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node4/data/tlog",
          "core":"solrj_collection3_shard1_replica_n1",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{"core_node8":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node8/data/",
          "base_url":"http://127.0.0.1:41995/_mhe/ly",
          "node_name":"127.0.0.1:41995__mhe%2Fly",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node8/data/tlog",
          "core":"solrj_collection3_shard2_replica_n2",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{"core_node6":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node6/data/",
          "base_url":"http://127.0.0.1:41995/_mhe/ly",
          "node_name":"127.0.0.1:41995__mhe%2Fly",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node6/data/tlog",
          "core":"solrj_collection3_shard3_replica_n3",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{"core_node9":{
          "core":"solrj_collection3_shard4_replica_n5",
          "base_url":"http://127.0.0.1:41995/_mhe/ly",
          "node_name":"127.0.0.1:41995__mhe%2Fly",
          "state":"down",
          "type":"NRT"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{"core_node10":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node10/data/",
          "base_url":"http://127.0.0.1:39323/_mhe/ly",
          "node_name":"127.0.0.1:39323__mhe%2Fly",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:43325/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",
          "core":"solrj_collection3_shard5_replica_n7",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}}},
  "router":{
    "field":"myOwnField",
    "name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"true",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:37530__mhe%2Fly, 127.0.0.1:39323__mhe%2Fly, 127.0.0.1:41313__mhe%2Fly, 127.0.0.1:41995__mhe%2Fly, 127.0.0.1:46306__mhe%2Fly]
	at __randomizedtesting.SeedInfo.seed([1E7F9EF5B5F4982:89B3C635F5A3247A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:413)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:255)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:143)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.test

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([1E7F9EF5B5F4982:89B3C635F5A3247A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.assertInvariants(TimeRoutedAliasUpdateProcessorTest.java:245)
	at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.test(TimeRoutedAliasUpdateProcessorTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 10900 lines...]
   [junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2> dic 13, 2017 10:02:32 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
   [junit4]   2>    1) Thread[id=16, name=TEST-TestGeo3DPoint.testRandomBig-seed#[EFC0B33DA0225E8E], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at sun.nio.ch.NativeThread.current(Native Method)
   [junit4]   2>         at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
   [junit4]   2>         at sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:737)
   [junit4]   2>         at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:727)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterFileChannel.read(FilterFileChannel.java:111)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterFileChannel.read(FilterFileChannel.java:111)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterFileChannel.read(FilterFileChannel.java:111)
   [junit4]   2>         at org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54)
   [junit4]   2>         at org.apache.lucene.store.DataInput.readInt(DataInput.java:102)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183)
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.readInt(MockIndexInputWrapper.java:165)
   [junit4]   2>         at org.apache.lucene.store.IndexInput$1.readInt(IndexInput.java:144)
   [junit4]   2>         at org.apache.lucene.util.packed.DirectReader$DirectPackedReader20.get(DirectReader.java:206)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$2.longValue(Lucene70DocValuesProducer.java:471)
   [junit4]   2>         at org.apache.lucene.index.AssertingLeafReader$AssertingNumericDocValues.longValue(AssertingLeafReader.java:478)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:840)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    2) Thread[id=15, name=SUITE-TestGeo3DPoint-seed#[EFC0B33DA0225E8E], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=EFC0B33DA0225E8E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-GT -Dtests.timezone=Asia/Saigon -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   6485s J1 | TestGeo3DPoint.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EFC0B33DA0225E8E]:0)
   [junit4]   2> dic 13, 2017 10:02:33 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> dic 13, 2017 10:02:53 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint: 
   [junit4]   2>    1) Thread[id=16, name=TEST-TestGeo3DPoint.testRandomBig-seed#[EFC0B33DA0225E8E], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$DualCrossingEdgeIterator.<init>(GeoComplexPolygon.java:993)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:366)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoBaseShape.getBounds(GeoBaseShape.java:46)
   [junit4]   2>         at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:441)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
   [junit4]   2>         at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> dic 13, 2017 10:02:53 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=16, name=TEST-TestGeo3DPoint.testRandomBig-seed#[EFC0B33DA0225E8E], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/spatial3d/test/J1/temp/lucene.spatial3d.TestGeo3DPoint_EFC0B33DA0225E8E-001
   [junit4]   2> Dec 13, 2017 3:02:53 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=LuceneFixedGap)}, docValues:{id=DocValuesFormat(name=Asserting), point=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1444, maxMBSortInHeap=6.034979220861164, sim=RandomSimilarity(queryNorm=true): {}, locale=es-GT, timezone=Asia/Saigon
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=121286160,total=469237760
   [junit4]   2> NOTE: All tests run in this JVM: [RandomGeoShapeRelationshipTest, GeoCircleTest, TestGeo3DPoint]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestGeo3DPoint -Dtests.seed=EFC0B33DA0225E8E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-GT -Dtests.timezone=Asia/Saigon -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestGeo3DPoint (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EFC0B33DA0225E8E]:0)
   [junit4] Completed [16/16 (1!)] on J1 in 7220.97s, 12 tests, 2 errors <<< FAILURES!

[...truncated 2066 lines...]
   [junit4] Suite: org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/init-core-data-001
   [junit4]   2> 1667634 WARN  (SUITE-TimeRoutedAliasUpdateProcessorTest-seed#[1E7F9EF5B5F4982]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=39 numCloses=39
   [junit4]   2> 1667634 INFO  (SUITE-TimeRoutedAliasUpdateProcessorTest-seed#[1E7F9EF5B5F4982]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1667635 INFO  (SUITE-TimeRoutedAliasUpdateProcessorTest-seed#[1E7F9EF5B5F4982]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1667635 INFO  (SUITE-TimeRoutedAliasUpdateProcessorTest-seed#[1E7F9EF5B5F4982]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1667636 INFO  (SUITE-TimeRoutedAliasUpdateProcessorTest-seed#[1E7F9EF5B5F4982]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001
   [junit4]   2> 1667654 INFO  (SUITE-TimeRoutedAliasUpdateProcessorTest-seed#[1E7F9EF5B5F4982]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1667722 INFO  (Thread-839) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1667722 INFO  (Thread-839) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1667743 ERROR (Thread-839) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1667832 INFO  (SUITE-TimeRoutedAliasUpdateProcessorTest-seed#[1E7F9EF5B5F4982]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:40495
   [junit4]   2> 1667999 INFO  (zkConnectionManagerCallback-539-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668264 INFO  (jetty-launcher-536-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1668343 INFO  (jetty-launcher-536-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@f53b8bb{/solr,null,AVAILABLE}
   [junit4]   2> 1668413 INFO  (jetty-launcher-536-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1668450 INFO  (jetty-launcher-536-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@608afa86{HTTP/1.1,[http/1.1]}{127.0.0.1:40305}
   [junit4]   2> 1668450 INFO  (jetty-launcher-536-thread-1) [    ] o.e.j.s.Server Started @1682915ms
   [junit4]   2> 1668450 INFO  (jetty-launcher-536-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40305}
   [junit4]   2> 1668451 ERROR (jetty-launcher-536-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1668451 INFO  (jetty-launcher-536-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1668451 INFO  (jetty-launcher-536-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1668451 INFO  (jetty-launcher-536-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1668451 INFO  (jetty-launcher-536-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-13T15:40:34.197Z
   [junit4]   2> 1668657 INFO  (jetty-launcher-536-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1cd9fa49{/solr,null,AVAILABLE}
   [junit4]   2> 1668657 INFO  (jetty-launcher-536-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1906dae{HTTP/1.1,[http/1.1]}{127.0.0.1:37806}
   [junit4]   2> 1668657 INFO  (jetty-launcher-536-thread-2) [    ] o.e.j.s.Server Started @1683122ms
   [junit4]   2> 1668657 INFO  (jetty-launcher-536-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37806}
   [junit4]   2> 1668658 ERROR (jetty-launcher-536-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1668658 INFO  (jetty-launcher-536-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.3.0
   [junit4]   2> 1668658 INFO  (jetty-launcher-536-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1668658 INFO  (jetty-launcher-536-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1668658 INFO  (jetty-launcher-536-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-13T15:40:34.404Z
   [junit4]   2> 1668733 INFO  (zkConnectionManagerCallback-541-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668733 INFO  (jetty-launcher-536-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1668752 INFO  (zkConnectionManagerCallback-543-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668759 INFO  (jetty-launcher-536-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1668874 INFO  (jetty-launcher-536-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40495/solr
   [junit4]   2> 1668875 INFO  (jetty-launcher-536-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40495/solr
   [junit4]   2> 1668934 INFO  (zkConnectionManagerCallback-549-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668971 INFO  (zkConnectionManagerCallback-551-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1668973 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x1605088d7400003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1669080 INFO  (zkConnectionManagerCallback-553-thread-1-processing-n:127.0.0.1:40305_solr) [n:127.0.0.1:40305_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1669133 INFO  (zkConnectionManagerCallback-555-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1670272 INFO  (jetty-launcher-536-thread-2) [n:127.0.0.1:37806_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1670293 INFO  (jetty-launcher-536-thread-2) [n:127.0.0.1:37806_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37806_solr
   [junit4]   2> 1670294 INFO  (jetty-launcher-536-thread-2) [n:127.0.0.1:37806_solr    ] o.a.s.c.Overseer Overseer (id=99167740459220998-127.0.0.1:37806_solr-n_0000000000) starting
   [junit4]   2> 1670331 INFO  (jetty-launcher-536-thread-2) [n:127.0.0.1:37806_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37806_solr
   [junit4]   2> 1670417 INFO  (OverseerStateUpdate-99167740459220998-127.0.0.1:37806_solr-n_0000000000) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1670454 INFO  (jetty-launcher-536-thread-1) [n:127.0.0.1:40305_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1670455 INFO  (jetty-launcher-536-thread-1) [n:127.0.0.1:40305_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1670457 INFO  (jetty-launcher-536-thread-1) [n:127.0.0.1:40305_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40305_solr
   [junit4]   2> 1670529 INFO  (zkCallback-554-thread-2-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1670620 INFO  (zkCallback-552-thread-1-processing-n:127.0.0.1:40305_solr) [n:127.0.0.1:40305_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1671038 INFO  (jetty-launcher-536-thread-2) [n:127.0.0.1:37806_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37806.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4cb691c8
   [junit4]   2> 1671087 INFO  (jetty-launcher-536-thread-1) [n:127.0.0.1:40305_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40305.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4cb691c8
   [junit4]   2> 1671094 INFO  (jetty-launcher-536-thread-2) [n:127.0.0.1:37806_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37806.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4cb691c8
   [junit4]   2> 1671095 INFO  (jetty-launcher-536-thread-2) [n:127.0.0.1:37806_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37806.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4cb691c8
   [junit4]   2> 1671096 INFO  (jetty-launcher-536-thread-2) [n:127.0.0.1:37806_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/.
   [junit4]   2> 1671275 INFO  (jetty-launcher-536-thread-1) [n:127.0.0.1:40305_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40305.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4cb691c8
   [junit4]   2> 1671276 INFO  (jetty-launcher-536-thread-1) [n:127.0.0.1:40305_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40305.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4cb691c8
   [junit4]   2> 1671277 INFO  (jetty-launcher-536-thread-1) [n:127.0.0.1:40305_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node1/.
   [junit4]   2> 1671560 INFO  (zkConnectionManagerCallback-560-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1672114 INFO  (TEST-TimeRoutedAliasUpdateProcessorTest.test-seed#[1E7F9EF5B5F4982]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1672165 INFO  (zkConnectionManagerCallback-565-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1672166 INFO  (TEST-TimeRoutedAliasUpdateProcessorTest.test-seed#[1E7F9EF5B5F4982]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1672167 INFO  (TEST-TimeRoutedAliasUpdateProcessorTest.test-seed#[1E7F9EF5B5F4982]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40495/solr ready
   [junit4]   2> 1672185 INFO  (qtp204805047-3110) [n:127.0.0.1:37806_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&name=timeConfig&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1672531 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.CreateCollectionCmd Create collection timeConfig
   [junit4]   2> 1672548 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.CreateCollectionCmd Only _default config set found, using it.
   [junit4]   2> 1672548 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/protwords.txt to /configs/timeConfig/protwords.txt
   [junit4]   2> 1672549 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/managed-schema to /configs/timeConfig/managed-schema
   [junit4]   2> 1672551 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/solrconfig.xml to /configs/timeConfig/solrconfig.xml
   [junit4]   2> 1672564 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/synonyms.txt to /configs/timeConfig/synonyms.txt
   [junit4]   2> 1672565 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/stopwords.txt to /configs/timeConfig/stopwords.txt
   [junit4]   2> 1672566 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/params.json to /configs/timeConfig/params.json
   [junit4]   2> 1672567 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_it.txt to /configs/timeConfig/lang/contractions_it.txt
   [junit4]   2> 1672585 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_no.txt to /configs/timeConfig/lang/stopwords_no.txt
   [junit4]   2> 1672586 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hy.txt to /configs/timeConfig/lang/stopwords_hy.txt
   [junit4]   2> 1672587 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stemdict_nl.txt to /configs/timeConfig/lang/stemdict_nl.txt
   [junit4]   2> 1672588 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ca.txt to /configs/timeConfig/lang/contractions_ca.txt
   [junit4]   2> 1672601 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_id.txt to /configs/timeConfig/lang/stopwords_id.txt
   [junit4]   2> 1672602 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ga.txt to /configs/timeConfig/lang/stopwords_ga.txt
   [junit4]   2> 1672603 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_da.txt to /configs/timeConfig/lang/stopwords_da.txt
   [junit4]   2> 1672621 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hi.txt to /configs/timeConfig/lang/stopwords_hi.txt
   [junit4]   2> 1672622 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ja.txt to /configs/timeConfig/lang/stopwords_ja.txt
   [junit4]   2> 1672623 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_pt.txt to /configs/timeConfig/lang/stopwords_pt.txt
   [junit4]   2> 1672624 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hu.txt to /configs/timeConfig/lang/stopwords_hu.txt
   [junit4]   2> 1672659 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_el.txt to /configs/timeConfig/lang/stopwords_el.txt
   [junit4]   2> 1672661 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ru.txt to /configs/timeConfig/lang/stopwords_ru.txt
   [junit4]   2> 1672748 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_tr.txt to /configs/timeConfig/lang/stopwords_tr.txt
   [junit4]   2> 1672784 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ar.txt to /configs/timeConfig/lang/stopwords_ar.txt
   [junit4]   2> 1672785 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/userdict_ja.txt to /configs/timeConfig/lang/userdict_ja.txt
   [junit4]   2> 1672786 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_it.txt to /configs/timeConfig/lang/stopwords_it.txt
   [junit4]   2> 1672820 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_eu.txt to /configs/timeConfig/lang/stopwords_eu.txt
   [junit4]   2> 1672821 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_cz.txt to /configs/timeConfig/lang/stopwords_cz.txt
   [junit4]   2> 1672822 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stoptags_ja.txt to /configs/timeConfig/lang/stoptags_ja.txt
   [junit4]   2> 1672823 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_fr.txt to /configs/timeConfig/lang/contractions_fr.txt
   [junit4]   2> 1672837 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fa.txt to /configs/timeConfig/lang/stopwords_fa.txt
   [junit4]   2> 1672839 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_de.txt to /configs/timeConfig/lang/stopwords_de.txt
   [junit4]   2> 1672856 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ga.txt to /configs/timeConfig/lang/contractions_ga.txt
   [junit4]   2> 1672857 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ca.txt to /configs/timeConfig/lang/stopwords_ca.txt
   [junit4]   2> 1672858 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_nl.txt to /configs/timeConfig/lang/stopwords_nl.txt
   [junit4]   2> 1672859 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_bg.txt to /configs/timeConfig/lang/stopwords_bg.txt
   [junit4]   2> 1672893 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_es.txt to /configs/timeConfig/lang/stopwords_es.txt
   [junit4]   2> 1672894 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_en.txt to /configs/timeConfig/lang/stopwords_en.txt
   [junit4]   2> 1672895 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_lv.txt to /configs/timeConfig/lang/stopwords_lv.txt
   [junit4]   2> 1672909 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_th.txt to /configs/timeConfig/lang/stopwords_th.txt
   [junit4]   2> 1672910 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ro.txt to /configs/timeConfig/lang/stopwords_ro.txt
   [junit4]   2> 1672911 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fr.txt to /configs/timeConfig/lang/stopwords_fr.txt
   [junit4]   2> 1672912 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_sv.txt to /configs/timeConfig/lang/stopwords_sv.txt
   [junit4]   2> 1672946 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fi.txt to /configs/timeConfig/lang/stopwords_fi.txt
   [junit4]   2> 1672947 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/hyphenations_ga.txt to /configs/timeConfig/lang/hyphenations_ga.txt
   [junit4]   2> 1672948 INFO  (OverseerThreadFactory-975-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_gl.txt to /configs/timeConfig/lang/stopwords_gl.txt
   [junit4]   2> 1673357 INFO  (OverseerStateUpdate-99167740459220998-127.0.0.1:37806_solr-n_0000000000) [n:127.0.0.1:37806_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"timeConfig",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"timeConfig_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:37806/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1673743 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=timeConfig&newCollection=true&name=timeConfig_shard1_replica_n1&action=CREATE&numShards=1&collection=timeConfig&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1673743 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1674031 INFO  (zkCallback-554-thread-2-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/timeConfig/state.json] for collection [timeConfig] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1674031 INFO  (zkCallback-554-thread-1-processing-n:127.0.0.1:37806_solr) [n:127.0.0.1:37806_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/timeConfig/state.json] for collection [timeConfig] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1675172 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.RequestParams conf resource params.json loaded . version : 0 
   [junit4]   2> 1675173 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.RequestParams request params refreshed to version 0
   [junit4]   2> 1675174 WARN  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../contrib/extraction/lib filtered by .*\.jar to classpath: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../contrib/extraction/lib
   [junit4]   2> 1675226 WARN  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../dist filtered by solr-cell-\d.*\.jar to classpath: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../dist
   [junit4]   2> 1675227 WARN  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../contrib/clustering/lib filtered by .*\.jar to classpath: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../contrib/clustering/lib
   [junit4]   2> 1675227 WARN  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../dist filtered by solr-clustering-\d.*\.jar to classpath: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../dist
   [junit4]   2> 1675227 WARN  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../contrib/langid/lib filtered by .*\.jar to classpath: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../contrib/langid/lib
   [junit4]   2> 1675227 WARN  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../dist filtered by solr-langid-\d.*\.jar to classpath: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../dist
   [junit4]   2> 1675227 WARN  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../contrib/velocity/lib filtered by .*\.jar to classpath: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../contrib/velocity/lib
   [junit4]   2> 1675228 WARN  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Couldn't add files from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../dist filtered by solr-velocity-\d.*\.jar to classpath: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1/../../../../dist
   [junit4]   2> 1675321 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
   [junit4]   2> 1675922 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.s.IndexSchema [timeConfig_shard1_replica_n1] Schema name=default-config
   [junit4]   2> 1677932 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema default-config/1.6 with uniqueid field id
   [junit4]   2> 1678296 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'timeConfig_shard1_replica_n1' using configuration from collection timeConfig, trusted=true
   [junit4]   2> 1678297 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37806.solr.core.timeConfig.shard1.replica_n1' (registry 'solr.core.timeConfig.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4cb691c8
   [junit4]   2> 1678297 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1678297 INFO  (qtp204805047-3108) [n:127.0.0.1:37806_solr c:timeConfig s:shard1 r:core_node2 x:timeConfig_shard1_replica_n1] o.a.s.c.SolrCore [[timeConfig_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_1E7F9EF5B5F4982-001/tempDir-001/node2/timeConfig_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyT

[...truncated too long message...]

tate change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 11212789 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[1E7F9EF5B5F4982]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:42113_
   [junit4]   2> 11212790 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[1E7F9EF5B5F4982]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 11212791 INFO  (zkCallback-3711-thread-2-processing-n:127.0.0.1:42113_) [n:127.0.0.1:42113_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 11212825 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[1E7F9EF5B5F4982]) [    ] o.a.s.c.Overseer Overseer (id=99168362789404690-127.0.0.1:42113_-n_0000000004) closing
   [junit4]   2> 11212825 WARN  (OverseerAutoScalingTriggerThread-99168362789404690-127.0.0.1:42113_-n_0000000004) [n:127.0.0.1:42113_    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 11212863 INFO  (OverseerStateUpdate-99168362789404690-127.0.0.1:42113_-n_0000000004) [n:127.0.0.1:42113_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:42113_
   [junit4]   2> 11213330 WARN  (zkCallback-3711-thread-1-processing-n:127.0.0.1:42113_) [n:127.0.0.1:42113_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_n47] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 11213330 INFO  (zkCallback-3711-thread-1-processing-n:127.0.0.1:42113_) [n:127.0.0.1:42113_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_n47] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 11213330 INFO  (zkCallback-3711-thread-1-processing-n:127.0.0.1:42113_) [n:127.0.0.1:42113_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_n47] o.a.s.c.SolrCore [collection1_shard1_replica_n47]  CLOSING SolrCore org.apache.solr.core.SolrCore@8232b74
   [junit4]   2> 11213330 INFO  (zkCallback-3711-thread-1-processing-n:127.0.0.1:42113_) [n:127.0.0.1:42113_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_n47] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.replica_n47, tag=136522612
   [junit4]   2> 11213330 INFO  (zkCallback-3711-thread-1-processing-n:127.0.0.1:42113_) [n:127.0.0.1:42113_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_n47] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@613f3365: rootName = null, domain = solr.core.collection1.shard1.replica_n47, service url = null, agent id = null] for registry solr.core.collection1.shard1.replica_n47 / com.codahale.metrics.MetricRegistry@515b3ba
   [junit4]   2> 11213369 INFO  (zkCallback-3711-thread-1-processing-n:127.0.0.1:42113_) [n:127.0.0.1:42113_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_n47] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=136522612
   [junit4]   2> 11213488 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[1E7F9EF5B5F4982]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@18358e8b{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 11213488 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[1E7F9EF5B5F4982]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@665b42ba{/,null,UNAVAILABLE}
   [junit4]   2> 11213524 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[1E7F9EF5B5F4982]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 11213527 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[1E7F9EF5B5F4982]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39871 39871
   [junit4]   2> 11213759 INFO  (Thread-6959) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39871 39871
   [junit4]   2> 11213763 WARN  (Thread-6959) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/aliases.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	5	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/clusterstate.json
   [junit4]   2> 	7	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/autoscaling.json
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/live_nodes
   [junit4]   2> 	7	/solr/collections
   [junit4]   2> 	5	/solr/overseer/queue
   [junit4]   2> 	5	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 	5	/solr/overseer/collection-queue-work
   [junit4]   2> 	5	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ShardSplitTest -Dtests.method=testSplitAfterFailedSplit -Dtests.seed=1E7F9EF5B5F4982 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=id -Dtests.timezone=Brazil/East -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 50.5s J1 | ShardSplitTest.testSplitAfterFailedSplit <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<1> but was:<2>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1E7F9EF5B5F4982:F8AA6A40672A0408]:0)
   [junit4]    > 	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:279)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_1E7F9EF5B5F4982-001
   [junit4]   2> Dec 13, 2017 6:19:39 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=id, timezone=Brazil/East
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=208344120,total=527433728
   [junit4]   2> NOTE: All tests run in this JVM: [BinaryUpdateRequestHandlerTest, TestPerFieldSimilarity, TestAtomicUpdateErrorCases, TlogReplayBufferedWhileIndexingTest, ConcurrentDeleteAndCreateCollectionTest, ShufflingReplicaListTransformerTest, CdcrReplicationHandlerTest, ZkFailoverTest, TestCloudNestedDocsSort, DistributedTermsComponentTest, PeerSyncWithIndexFingerprintCachingTest, AssignTest, TestSubQueryTransformerCrossCore, FacetPivotSmallTest, TestRestoreCore, HdfsUnloadDistributedZkTest, TestInPlaceUpdatesStandalone, TestExclusionRuleCollectionAccess, TestFaceting, NumericFieldsTest, TestPKIAuthenticationPlugin, DocumentAnalysisRequestHandlerTest, TestManagedSynonymGraphFilterFactory, SampleTest, SparseHLLTest, TestWriterPerf, RegexBoostProcessorTest, TestLuceneMatchVersion, JSONWriterTest, TestStressReorder, SynonymTokenizerTest, CdcrBidirectionalTest, TestFastWriter, TestBlendedInfixSuggestions, SubstringBytesRefFilterTest, TestSlowCompositeReaderWrapper, TestNonDefinedSimilarityFactory, CursorMarkTest, TestCustomSort, TestFieldCacheWithThreads, TemplateUpdateProcessorTest, BasicAuthStandaloneTest, SolrIndexConfigTest, RollingRestartTest, TestManagedResourceStorage, TestDocSet, TestReplicaProperties, ChangedSchemaMergeTest, TestInfoStreamLogging, TestTolerantUpdateProcessorCloud, LukeRequestHandlerTest, PrimUtilsTest, CoreAdminCreateDiscoverTest, OutputWriterTest, TestBulkSchemaConcurrent, JavabinLoaderTest, DistributedFacetExistsSmallTest, MultiThreadedOCPTest, SolrRequestParserTest, TestDocTermOrds, SolrCloudReportersTest, BadIndexSchemaTest, TestRemoteStreaming, PingRequestHandlerTest, SimpleMLTQParserTest, TestRawResponseWriter, SignatureUpdateProcessorFactoryTest, TestZkAclsWithHadoopAuth, TestFastLRUCache, TestSolr4Spatial, ConjunctionSolrSpellCheckerTest, CollectionTooManyReplicasTest, SolrMetricsIntegrationTest, TestSurroundQueryParser, ParsingFieldUpdateProcessorsTest, SuggesterTSTTest, ConfigureRecoveryStrategyTest, TestCoreAdminApis, BlockJoinFacetRandomTest, NodeLostTriggerTest, OpenCloseCoreStressTest, MoveReplicaHDFSFailoverTest, TestFieldTypeResource, SuggesterFSTTest, TestJettySolrRunner, DOMUtilTest, PreAnalyzedFieldManagedSchemaCloudTest, TestXmlQParserPlugin, TestRangeQuery, TestCloudRecovery, TestStressUserVersions, ChaosMonkeySafeLeaderTest, DeleteLastCustomShardedReplicaTest, TestOmitPositions, ClassificationUpdateProcessorFactoryTest, ShowFileRequestHandlerTest, HdfsSyncSliceTest, SolrJmxReporterCloudTest, TestQuerySenderNoQuery, SolrCLIZkUtilsTest, TestSolrConfigHandler, TestLegacyNumericRangeQueryBuilder, CreateCollectionCleanupTest, TestSizeLimitedDistributedMap, ShardSplitTest]
   [junit4] Completed [550/752 (6!)] on J1 in 670.96s, 5 tests, 1 failure <<< FAILURES!

[...truncated 48044 lines...]