You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/07/11 11:09:06 UTC

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1582 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1582/

3 tests failed.
FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/28)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{"core_node9":{           "dataDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node9/data/",           "base_url":"http://127.0.0.1:42112/u_/m",           "node_name":"127.0.0.1:42112_u_%2Fm",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",           "core":"solrj_collection3_shard1_replica_n1",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{"core_node5":{           "dataDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node5/data/",           "base_url":"http://127.0.0.1:43778/u_/m",           "node_name":"127.0.0.1:43778_u_%2Fm",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node5/data/tlog",           "core":"solrj_collection3_shard2_replica_n2",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{"core_node7":{           "dataDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node7/data/",           "base_url":"http://127.0.0.1:43778/u_/m",           "node_name":"127.0.0.1:43778_u_%2Fm",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node7/data/tlog",           "core":"solrj_collection3_shard3_replica_n3",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{"core_node8":{           "core":"solrj_collection3_shard4_replica_n4",           "base_url":"http://127.0.0.1:43778/u_/m",           "node_name":"127.0.0.1:43778_u_%2Fm",           "state":"down",           "type":"NRT",           "leader":"true"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{"core_node10":{           "dataDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node10/data/",           "base_url":"http://127.0.0.1:43778/u_/m",           "node_name":"127.0.0.1:43778_u_%2Fm",           "type":"NRT",           "ulogDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",           "core":"solrj_collection3_shard5_replica_n6",           "shared_storage":"true",           "state":"active",           "leader":"true"}}}},   "router":{     "field":"myOwnField",     "name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"true",   "nrtReplicas":"1",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:42112_u_%2Fm, 127.0.0.1:43778_u_%2Fm, 127.0.0.1:45011_u_%2Fm, 127.0.0.1:45426_u_%2Fm]

Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/28)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{"core_node9":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node9/data/",
          "base_url":"http://127.0.0.1:42112/u_/m",
          "node_name":"127.0.0.1:42112_u_%2Fm",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node9/data/tlog",
          "core":"solrj_collection3_shard1_replica_n1",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{"core_node5":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node5/data/",
          "base_url":"http://127.0.0.1:43778/u_/m",
          "node_name":"127.0.0.1:43778_u_%2Fm",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node5/data/tlog",
          "core":"solrj_collection3_shard2_replica_n2",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{"core_node7":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node7/data/",
          "base_url":"http://127.0.0.1:43778/u_/m",
          "node_name":"127.0.0.1:43778_u_%2Fm",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node7/data/tlog",
          "core":"solrj_collection3_shard3_replica_n3",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{"core_node8":{
          "core":"solrj_collection3_shard4_replica_n4",
          "base_url":"http://127.0.0.1:43778/u_/m",
          "node_name":"127.0.0.1:43778_u_%2Fm",
          "state":"down",
          "type":"NRT",
          "leader":"true"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{"core_node10":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node10/data/",
          "base_url":"http://127.0.0.1:43778/u_/m",
          "node_name":"127.0.0.1:43778_u_%2Fm",
          "type":"NRT",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/solrj_collection3/core_node10/data/tlog",
          "core":"solrj_collection3_shard5_replica_n6",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}}},
  "router":{
    "field":"myOwnField",
    "name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"true",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:42112_u_%2Fm, 127.0.0.1:43778_u_%2Fm, 127.0.0.1:45011_u_%2Fm, 127.0.0.1:45426_u_%2Fm]
	at __randomizedtesting.SeedInfo.seed([E49AD33B821B12EB:6CCEECE12CE77F13]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:415)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:257)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
KeeperErrorCode = Session expired for /live_nodes

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /live_nodes
	at __randomizedtesting.SeedInfo.seed([E49AD33B821B12EB:F181AEFA01ECEA90]:0)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1535)
	at org.apache.solr.common.cloud.SolrZkClient.lambda$getChildren$4(SolrZkClient.java:329)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:329)
	at org.apache.solr.common.cloud.ZkStateReader.refreshLiveNodes(ZkStateReader.java:762)
	at org.apache.solr.common.cloud.ZkStateReader.updateLiveNodes(ZkStateReader.java:405)
	at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=21233, name=Connection evictor, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=21233, name=Connection evictor, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded




Build Log:
[...truncated 14864 lines...]
   [junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/init-core-data-001
   [junit4]   2> 4691107 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=91 numCloses=91
   [junit4]   2> 4691107 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 4691121 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 4691121 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 4691122 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /u_/m
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 4691437 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 4691560 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4691562 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4691739 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_45977_hdfs____nxbjtu/webapp
   [junit4]   2> 4693596 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:45977
   [junit4]   2> 4695091 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4695141 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4695419 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_38672_datanode____4kmnsj/webapp
   [junit4]   2> 4696942 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38672
   [junit4]   2> 4698107 WARN  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4698108 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4698372 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_33029_datanode____.nxm83m/webapp
   [junit4]   2> 4699729 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:34630) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 4700328 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x30741a2faa4577: from storage DS-e2405150-1b2e-4f97-a31b-ccdb96e0062a node DatanodeRegistration(127.0.0.1:33261, datanodeUuid=d38f06a2-23a7-408b-b9a3-aa45760ca0e8, infoPort=40547, infoSecurePort=0, ipcPort=40071, storageInfo=lv=-56;cid=testClusterID;nsid=362930972;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 4700328 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x30741a2faa4577: from storage DS-4c15b0ff-12b4-4f74-beca-006d55bfe5ec node DatanodeRegistration(127.0.0.1:33261, datanodeUuid=d38f06a2-23a7-408b-b9a3-aa45760ca0e8, infoPort=40547, infoSecurePort=0, ipcPort=40071, storageInfo=lv=-56;cid=testClusterID;nsid=362930972;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 4700876 INFO  (SUITE-SharedFSAutoReplicaFailoverTest-seed#[E49AD33B821B12EB]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33029
   [junit4]   2> 4702689 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to lucene2-us-west.apache.org/127.0.0.1:34630) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 4702819 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x30741ac42b16b1: from storage DS-8862805b-c6da-44e9-a89b-6b1d0b1285b7 node DatanodeRegistration(127.0.0.1:43230, datanodeUuid=2ecb2f15-dc97-4743-adb5-adb71ad90815, infoPort=36725, infoSecurePort=0, ipcPort=41205, storageInfo=lv=-56;cid=testClusterID;nsid=362930972;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 4702819 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x30741ac42b16b1: from storage DS-f5411200-0bb1-4da8-bf93-812cd4faea5b node DatanodeRegistration(127.0.0.1:43230, datanodeUuid=2ecb2f15-dc97-4743-adb5-adb71ad90815, infoPort=36725, infoSecurePort=0, ipcPort=41205, storageInfo=lv=-56;cid=testClusterID;nsid=362930972;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 4703768 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4703800 INFO  (Thread-51927) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4703800 INFO  (Thread-51927) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4703839 ERROR (Thread-51927) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4703896 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.ZkTestServer start zk server on port:44994
   [junit4]   2> 4704024 INFO  (zkConnectionManagerCallback-4832-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4704150 INFO  (zkConnectionManagerCallback-4834-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4704153 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4704167 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 4704168 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4704169 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4704207 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4704208 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4704209 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4704210 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4704223 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4704224 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4704225 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4704226 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 4704994 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 4705095 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4705095 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4705095 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4705136 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a971065{/u_/m,null,AVAILABLE}
   [junit4]   2> 4705136 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@20c3dce0{HTTP/1.1,[http/1.1]}{127.0.0.1:45011}
   [junit4]   2> 4705136 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.Server Started @4705273ms
   [junit4]   2> 4705136 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, hostContext=/u_/m, hostPort=45011, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/control-001/cores}
   [junit4]   2> 4705136 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4705136 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4705137 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 4705137 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4705137 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4705137 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-11T09:24:18.572Z
   [junit4]   2> 4705243 INFO  (zkConnectionManagerCallback-4836-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4705337 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4705337 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/control-001/solr.xml
   [junit4]   2> 4705341 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4705341 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4705374 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4706377 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44994/solr
   [junit4]   2> 4706410 INFO  (zkConnectionManagerCallback-4840-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4706467 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1032cea5b610003, likely client has closed socket
   [junit4]   2> 4706544 INFO  (zkConnectionManagerCallback-4842-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4707027 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4707027 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45011_u_%2Fm
   [junit4]   2> 4707028 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.c.Overseer Overseer (id=72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) starting
   [junit4]   2> 4707216 INFO  (zkConnectionManagerCallback-4849-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4707342 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44994/solr ready
   [junit4]   2> 4707436 DEBUG (OverseerAutoScalingTriggerThread-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 4707451 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45011_u_%2Fm
   [junit4]   2> 4707472 DEBUG (OverseerAutoScalingTriggerThread-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 4707472 DEBUG (OverseerAutoScalingTriggerThread-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 4707472 DEBUG (OverseerAutoScalingTriggerThread-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 4707550 INFO  (OverseerStateUpdate-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4707647 INFO  (zkCallback-4848-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4707669 DEBUG (OverseerAutoScalingTriggerThread-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:45011_u_%2Fm]
   [junit4]   2> 4707699 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4707756 DEBUG (OverseerAutoScalingTriggerThread-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 4707757 DEBUG (OverseerAutoScalingTriggerThread-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 4707769 DEBUG (ScheduledTrigger-6792-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4707843 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4707899 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4707899 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4707901 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/control-001/cores
   [junit4]   2> 4708098 INFO  (zkConnectionManagerCallback-4855-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4708155 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4708160 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44994/solr ready
   [junit4]   2> 4708192 INFO  (qtp1280717846-66272) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45011_u_%252Fm&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4708291 INFO  (OverseerThreadFactory-6794-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 4708448 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm    x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4708449 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm    x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4708944 DEBUG (ScheduledTrigger-6792-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4709951 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4710401 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 4710523 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 4711034 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4711129 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 4711199 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 4711232 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4711232 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home
   [junit4]   2> 4711232 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 4711232 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4711233 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 4711234 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 4711307 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4711307 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4711365 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4711398 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 4711582 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 4711635 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4711635 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4711727 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4711728 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=0.5244140625, floorSegmentMB=1.6767578125, forceMergeDeletesPctAllowed=2.476511852767734, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0, reclaimDeletesWeight=0.3100030032171759
   [junit4]   2> 4712036 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4712201 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43230 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-4c15b0ff-12b4-4f74-beca-006d55bfe5ec:NORMAL:127.0.0.1:33261|RBW], ReplicaUC[[DISK]DS-f5411200-0bb1-4da8-bf93-812cd4faea5b:NORMAL:127.0.0.1:43230|FINALIZED]]} size 0
   [junit4]   2> 4712206 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33261 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f5411200-0bb1-4da8-bf93-812cd4faea5b:NORMAL:127.0.0.1:43230|FINALIZED], ReplicaUC[[DISK]DS-e2405150-1b2e-4f97-a31b-ccdb96e0062a:NORMAL:127.0.0.1:33261|FINALIZED]]} size 0
   [junit4]   2> 4712313 WARN  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 4712657 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 4712657 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4712657 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 4712756 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4712756 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4712771 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 4712898 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@45b32c47[control_collection_shard1_replica_n1] main]
   [junit4]   2> 4712899 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4712899 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4712900 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4712901 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1605685546934337536
   [junit4]   2> 4712932 INFO  (searcherExecutor-6799-thread-1-processing-n:127.0.0.1:45011_u_%2Fm x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@45b32c47[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4712937 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 4712952 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4712952 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4712952 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45011/u_/m/control_collection_shard1_replica_n1/
   [junit4]   2> 4712953 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4712953 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45011/u_/m/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 4712953 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 4712988 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45011/u_/m/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 4713044 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4713045 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4713047 INFO  (qtp1280717846-66267) [n:127.0.0.1:45011_u_%2Fm c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=4599
   [junit4]   2> 4713084 INFO  (qtp1280717846-66272) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 4713155 INFO  (zkCallback-4841-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4714045 DEBUG (ScheduledTrigger-6792-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4714089 INFO  (qtp1280717846-66272) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45011_u_%252Fm&wt=javabin&version=2} status=0 QTime=5897
   [junit4]   2> 4714292 INFO  (zkConnectionManagerCallback-4860-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4714330 INFO  (OverseerCollectionConfigSetProcessor-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4714330 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4714331 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44994/solr ready
   [junit4]   2> 4714331 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 4714332 INFO  (qtp1280717846-66268) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4714382 INFO  (OverseerThreadFactory-6794-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 4714383 WARN  (OverseerThreadFactory-6794-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 4714639 INFO  (qtp1280717846-66268) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 4714640 INFO  (qtp1280717846-66268) [n:127.0.0.1:45011_u_%2Fm    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=308
   [junit4]   2> 4715056 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 4715517 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-1-001 of type NRT
   [junit4]   2> 4715530 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 4715531 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4715531 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4715531 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@421a53a6{/u_/m,null,AVAILABLE}
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@35ce03e0{HTTP/1.1,[http/1.1]}{127.0.0.1:43778}
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.Server Started @4715669ms
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/u_/m, hostPort=43778, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-1-001/cores}
   [junit4]   2> 4715532 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4715532 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-11T09:24:28.967Z
   [junit4]   2> 4715551 INFO  (zkConnectionManagerCallback-4862-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4715552 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4715552 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-1-001/solr.xml
   [junit4]   2> 4715568 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4715568 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4715569 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4715938 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44994/solr
   [junit4]   2> 4715973 INFO  (zkConnectionManagerCallback-4866-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4715975 INFO  (zkConnectionManagerCallback-4868-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4715992 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4716010 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4716011 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4716012 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43778_u_%2Fm
   [junit4]   2> 4716037 INFO  (zkCallback-4841-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4716049 INFO  (zkCallback-4848-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4716173 INFO  (zkCallback-4867-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4716173 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 4716179 INFO  (zkCallback-4859-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4716403 INFO  (OverseerCollectionConfigSetProcessor-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4716479 INFO  (zkConnectionManagerCallback-4875-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4716513 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 4716528 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44994/solr ready
   [junit4]   2> 4716565 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4716639 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4716678 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4716678 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4716712 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-1-001/cores
   [junit4]   2> 4716987 INFO  (qtp323678340-66337) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:43778_u_%252Fm&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4717020 INFO  (OverseerThreadFactory-6794-thread-3) [ c:collection1 s:shard2  ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:43778_u_%2Fm for creating new replica
   [junit4]   2> 4717032 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm    x:collection1_shard2_replica_n41] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4717186 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 4717217 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 4717299 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
   [junit4]   2> 4717900 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 4717982 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
   [junit4]   2> 4717983 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4717983 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home
   [junit4]   2> 4717983 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 4717983 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4717983 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/collection1/core_node42/data/]
   [junit4]   2> 4717985 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/collection1/core_node42/data/snapshot_metadata
   [junit4]   2> 4718057 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4718057 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4718081 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4718082 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/collection1/core_node42/data
   [junit4]   2> 4718190 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 4718190 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34630/solr_hdfs_home/collection1/core_node42/data/index
   [junit4]   2> 4718208 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4718208 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
   [junit4]   2> 4718232 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4718232 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=0.5244140625, floorSegmentMB=1.6767578125, forceMergeDeletesPctAllowed=2.476511852767734, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0, reclaimDeletesWeight=0.3100030032171759
   [junit4]   2> 4718604 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33261 is added to blk_1073741826_1002{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e2405150-1b2e-4f97-a31b-ccdb96e0062a:NORMAL:127.0.0.1:33261|RBW], ReplicaUC[[DISK]DS-f5411200-0bb1-4da8-bf93-812cd4faea5b:NORMAL:127.0.0.1:43230|RBW]]} size 69
   [junit4]   2> 4718604 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43230 is added to blk_1073741826_1002 size 69
   [junit4]   2> 4719057 WARN  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 4719190 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 4719621 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 4719621 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4719621 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 4719674 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4719674 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4719711 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 4719890 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@1e73c65f[collection1_shard2_replica_n41] main]
   [junit4]   2> 4719911 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4719911 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4719912 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4719913 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1605685554286952448
   [junit4]   2> 4719938 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node42=0}, version=0}
   [junit4]   2> 4719948 INFO  (searcherExecutor-6813-thread-1-processing-n:127.0.0.1:43778_u_%2Fm x:collection1_shard2_replica_n41 c:collection1 s:shard2) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@1e73c65f[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4719973 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4719973 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4719973 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43778/u_/m/collection1_shard2_replica_n41/
   [junit4]   2> 4719973 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4719973 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:43778/u_/m/collection1_shard2_replica_n41/ has no replicas
   [junit4]   2> 4719973 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 4719993 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43778/u_/m/collection1_shard2_replica_n41/ shard2
   [junit4]   2> 4720045 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4720047 INFO  (qtp323678340-66332) [n:127.0.0.1:43778_u_%2Fm c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3015
   [junit4]   2> 4720167 INFO  (qtp323678340-66337) [n:127.0.0.1:43778_u_%2Fm c:collection1   ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:43778_u_%252Fm&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=3179
   [junit4]   2> 4720167 INFO  (zkCallback-4867-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 4720191 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 4720823 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-2-001 of type NRT
   [junit4]   2> 4720823 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 4720840 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4720840 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4720840 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4720841 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@378cf1cd{/u_/m,null,AVAILABLE}
   [junit4]   2> 4720842 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@23e5e0e5{HTTP/1.1,[http/1.1]}{127.0.0.1:44614}
   [junit4]   2> 4720842 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.e.j.s.Server Started @4720980ms
   [junit4]   2> 4720842 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/u_/m, hostPort=44614, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-2-001/cores}
   [junit4]   2> 4720843 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4720843 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4720843 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 4720843 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4720843 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4720843 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-11T09:24:34.278Z
   [junit4]   2> 4720857 INFO  (zkConnectionManagerCallback-4878-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4720858 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1032cea5b61000c, likely client has closed socket
   [junit4]   2> 4720858 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4720858 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-2-001/solr.xml
   [junit4]   2> 4720878 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4720878 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4720879 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4721068 INFO  (OverseerCollectionConfigSetProcessor-72951404035112964-127.0.0.1:45011_u_%2Fm-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4721194 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 4721903 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44994/solr
   [junit4]   2> 4722024 INFO  (zkConnectionManagerCallback-4882-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4722077 INFO  (zkConnectionManagerCallback-4884-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4722102 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 4722103 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4722105 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4722130 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44614_u_%2Fm
   [junit4]   2> 4722268 INFO  (zkCallback-4883-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4722296 INFO  (zkCallback-4859-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4722305 INFO  (zkCallback-4848-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4722305 DEBUG (ScheduledTrigger-6792-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 4722305 INFO  (zkCallback-4841-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4722306 INFO  (zkCallback-4867-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4722341 INFO  (zkCallback-4874-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 4722390 INFO  (zkConnectionManagerCallback-4891-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4722414 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 4722415 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44994/solr ready
   [junit4]   2> 4722446 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4722574 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4722649 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4722666 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7434cd46
   [junit4]   2> 4722667 INFO  (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[E49AD33B821B12EB]) [n:127.0.0.1:44614_u_%2Fm    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_E49AD33B821B12EB-001/shard-2-001/cores
   [junit4]   2> 4722980 INFO  (qtp323678340-66333) [n:127.0.0.1:43778_u_%2Fm    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:44614_u_%252Fm&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4722998 INFO  (OverseerThreadFactory-6794-thread-4) [ c:collection1 s:shard1  ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:44614_u_%2Fm for creating new replica
   [junit4]   2> 4723018 INFO  (qtp1951039649-66384) [n:127.0.0.1:44614_u_%2Fm    x:collection1_shard1_replica_n43] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4723044 INFO  (zkCallback-4867-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 4723092 INFO  (qtp1951039649-66384) [n:127.0.0.1:44614_u_%2Fm c:collection1 s:shard1  x:collection1_shard1_replica_n43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 4723145 INFO  (qtp1951039649-66

[...truncated too long message...]

gsFormat(name=MockRandom), 5child1_s=PostingsFormat(name=MockRandom), 14grand0_s=Lucene50(blocksize=128), 33parent_f2_s=Lucene50(blocksize=128), 13grand3_s=Lucene50(blocksize=128), 22parent_f2_s=Lucene50(blocksize=128), 4child0_s=BlockTreeOrds(blocksize=128), 11grand5_s=Lucene50(blocksize=128), 12grand2_s=Lucene50(blocksize=128), 12parent_f2_s=BlockTreeOrds(blocksize=128), 52parent_f2_s=BlockTreeOrds(blocksize=128), 42parent_f2_s=PostingsFormat(name=Asserting), 10grand4_s=Lucene50(blocksize=128), 32parent_f2_s=PostingsFormat(name=MockRandom), 21parent_f1_s=Lucene50(blocksize=128), 8grand2_s=PostingsFormat(name=MockRandom), 9grand1_s=PostingsFormat(name=MockRandom), 41parent_f1_s=PostingsFormat(name=Asserting), 12grand8_s=PostingsFormat(name=Asserting), 40parent_f2_s=Lucene50(blocksize=128), 47parent_f1_s=Lucene50(blocksize=128), 13grand7_s=Lucene50(blocksize=128), 14parent_f1_s=Lucene50(blocksize=128), 14grand10_s=BlockTreeOrds(blocksize=128), 20parent_f2_s=PostingsFormat(name=Asserting), 34parent_f1_s=PostingsFormat(name=Asserting), 6child4_s=PostingsFormat(name=MockRandom), 15grand9_s=Lucene50(blocksize=128), 6grand0_s=PostingsFormat(name=MockRandom), 7child3_s=PostingsFormat(name=MockRandom), 15grand0_s=BlockTreeOrds(blocksize=128), 8child6_s=PostingsFormat(name=MockRandom), 5child0_s=Lucene50(blocksize=128), 14grand1_s=BlockTreeOrds(blocksize=128), 8parent_f1_s=BlockTreeOrds(blocksize=128), 15grand13_s=PostingsFormat(name=MockRandom), 27parent_f2_s=BlockTreeOrds(blocksize=128), 2parent_f2_s=Lucene50(blocksize=128), 4child1_s=PostingsFormat(name=Asserting), 11grand4_s=BlockTreeOrds(blocksize=128), 16parent_f2_s=BlockTreeOrds(blocksize=128), 10grand5_s=BlockTreeOrds(blocksize=128), 36parent_f2_s=PostingsFormat(name=MockRandom), 13grand10_s=PostingsFormat(name=Asserting), 7grand6_s=PostingsFormat(name=Asserting), 18parent_f1_s=PostingsFormat(name=Asserting), 9grand8_s=Lucene50(blocksize=128), 8grand3_s=PostingsFormat(name=Asserting), 6parent_f1_s=PostingsFormat(name=MockRandom), 25parent_f1_s=Lucene50(blocksize=128), 12grand9_s=PostingsFormat(name=MockRandom), 0parent_f1_s=BlockTreeOrds(blocksize=128), 3grand2_s=PostingsFormat(name=Asserting), 9grand0_s=Lucene50(blocksize=128), 4parent_f2_s=PostingsFormat(name=Asserting), 47parent_f2_s=PostingsFormat(name=MockRandom), 13grand6_s=BlockTreeOrds(blocksize=128), 6child5_s=Lucene50(blocksize=128), 7child2_s=Lucene50(blocksize=128), 38parent_f1_s=PostingsFormat(name=Asserting), 5grand4_s=PostingsFormat(name=Asserting), 6grand1_s=PostingsFormat(name=Asserting), 15grand8_s=BlockTreeOrds(blocksize=128), 8child7_s=Lucene50(blocksize=128), 16parent_f1_s=Lucene50(blocksize=128), 14grand2_s=PostingsFormat(name=Asserting), 8parent_f2_s=PostingsFormat(name=Asserting), 15grand14_s=Lucene50(blocksize=128), 13grand5_s=PostingsFormat(name=Asserting), 29parent_f2_s=PostingsFormat(name=MockRandom), 2parent_f1_s=PostingsFormat(name=MockRandom), 4child2_s=PostingsFormat(name=MockRandom), rnd_b=PostingsFormat(name=Asserting), 11grand3_s=PostingsFormat(name=Asserting), 27parent_f1_s=PostingsFormat(name=Asserting), 45parent_f1_s=PostingsFormat(name=Asserting), 49parent_f2_s=BlockTreeOrds(blocksize=128), 12grand0_s=PostingsFormat(name=Asserting), 10grand6_s=PostingsFormat(name=Asserting), 2child0_s=PostingsFormat(name=MockRandom), 13grand11_s=PostingsFormat(name=MockRandom), 36parent_f1_s=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Lucene70), text_t=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1850, maxMBSortInHeap=7.589652052991062, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@7040ee3b), locale=es-PY, timezone=America/Panama
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=136302824,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [VersionInfoTest, CoreAdminCreateDiscoverTest, UninvertDocValuesMergePolicyTest, TestRTimerTree, SpellingQueryConverterTest, TestGeoJSONResponseWriter, HdfsRecoveryZkTest, DistributedQueryComponentCustomSortTest, MoveReplicaHDFSTest, MoreLikeThisHandlerTest, TestMiniSolrCloudClusterSSL, LoggingHandlerTest, PeerSyncWithLeaderAndIndexFingerprintCachingTest, RollingRestartTest, TestGroupingSearch, FileUtilsTest, ChangedSchemaMergeTest, DistributedDebugComponentTest, MultiThreadedOCPTest, TestTrackingShardHandlerFactory, SuggesterWFSTTest, RestoreTriggerStateTest, TestGraphTermsQParserPlugin, TestNamedUpdateProcessors, TestNodeLostTrigger, BlobRepositoryCloudTest, TestReqParamsAPI, MetricUtilsTest, DOMUtilTest, ZkControllerTest, TestSizeLimitedDistributedMap, AnalyticsMergeStrategyTest, PluginInfoTest, HttpPartitionTest, TestCustomSort, TestTrieFacet, TestTolerantUpdateProcessorCloud, CollectionsAPIDistributedZkTest, TestLuceneMatchVersion, TestSystemIdResolver, HdfsDirectoryTest, TestSearchPerf, ExitableDirectoryReaderTest, TestDFISimilarityFactory, TestAuthorizationFramework, BitVectorTest, SpatialFilterTest, TestExtendedDismaxParser, BadCopyFieldTest, TestCorePropertiesReload, TestXmlQParserPlugin, TestManagedSchemaThreadSafety, TestMultiWordSynonyms, TestWriterPerf, TestSchemaNameResource, TestExceedMaxTermLength, HdfsCollectionsAPIDistributedZkTest, TestStressLucene, AnalysisErrorHandlingTest, TestRestManager, TestLocalFSCloudBackupRestore, TestShardHandlerFactory, SolrCmdDistributorTest, TestLegacyNumericRangeQueryBuilder, RegexBytesRefFilterTest, TestSchemaVersionResource, DistributedFacetPivotLargeTest, TestExactSharedStatsCache, DateFieldTest, DocumentAnalysisRequestHandlerTest, ZkShardTermsTest, RegexBoostProcessorTest, TestPHPSerializedResponseWriter, RuleEngineTest, MigrateRouteKeyTest, TestComplexPhraseLeadingWildcard, TestCloudPseudoReturnFields, NodeLostTriggerIntegrationTest, TestCrossCoreJoin, TestSchemaManager, TestSolrCoreSnapshots, TransactionLogTest, ReplaceNodeTest, TestSmileRequest, TestHdfsBackupRestoreCore, DistanceFunctionTest, ShardRoutingCustomTest, TestConfigOverlay, ChaosMonkeyNothingIsSafeTest, TestConfigSetsAPIZkFailure, TestCloudSchemaless, FacetPivotSmallTest, RecoveryZkTest, CdcrRequestHandlerTest, TestFieldCacheSort, TestInitQParser, TestNumericRangeQuery32, DistribDocExpirationUpdateProcessorTest, IgnoreLargeDocumentProcessorFactoryTest, TestBlendedInfixSuggestions, ShufflingReplicaListTransformerTest, CreateCollectionCleanupTest, TestRawTransformer, SolrMetricsIntegrationTest, SolrJmxReporterCloudTest, PrimUtilsTest, TestLeaderElectionZkExpiry, ResponseHeaderTest, TestCloudJSONFacetJoinDomain, SuggesterFSTTest, SolrIndexSplitterTest, TestDocTermOrdsUninvertLimit, TestGenericDistributedQueue, TestMinMaxOnMultiValuedField, BlockJoinFacetRandomTest, TestStressInPlaceUpdates, DistributedSpellCheckComponentTest, TestRandomCollapseQParserPlugin, RequiredFieldsTest, EchoParamsTest, PeerSyncTest, TimeZoneUtilsTest, ZkStateReaderTest, CollectionReloadTest, ExternalFileFieldSortTest, SolrXmlInZkTest, HdfsBasicDistributedZk2Test, TestFieldResource, SolrCoreCheckLockOnStartupTest, TestCloudNestedDocsSort, AnalysisAfterCoreReloadTest, CursorPagingTest, MinimalSchemaTest, OutputWriterTest, SampleTest, SolrInfoBeanTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestDistributedGrouping, TestDistributedMissingSort, TestHighlightDedupGrouping, TestJoin, TestRandomDVFaceting, TestSolrCoreProperties, TestTolerantSearch, TestTrie, PathHierarchyTokenizerFactoryTest, TokenizerChainTest, TestLuceneIndexBackCompat, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, ConnectionReuseTest, ActionThrottleTest, AddReplicaTest, AliasIntegrationTest, AssignBackwardCompatibilityTest, AsyncCallRequestStatusResponseTest, BasicDistributedZk2Test, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ChaosMonkeySafeLeaderTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionPropsTest, ConcurrentCreateRoutedAliasTest, ConfigSetsAPITest, ConnectionManagerTest, CreateRoutedAliasTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteShardTest, DistributedVersionInfoTest, DocValuesNotIndexedTest, ForceLeaderTest, FullSolrCloudDistribCmdsTest]
   [junit4] Completed [526/823 (3!)] on J0 in 2280.24s, 1 test, 1 error <<< FAILURES!

[...truncated 1053 lines...]
   [junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20180711_080553_2185510846247313525781.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid10160.hprof ...
   [junit4] Heap dump file created [549103506 bytes in 40.500 secs]
   [junit4] <<< JVM J0: EOF ----

   [junit4] JVM J0: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20180711_080553_2182953064586134752412.syserr
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] 
   [junit4] Exception: java.lang.IllegalArgumentException thrown from the UncaughtExceptionHandler in thread "Connection evictor"
   [junit4] <<< JVM J0: EOF ----

[...truncated 8726 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:651: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid10160.hprof

Total time: 519 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1584 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1584/

1 tests failed.
FAILED:  org.apache.solr.update.TestInPlaceUpdatesDistrib.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:55775/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:55775/collection1
	at __randomizedtesting.SeedInfo.seed([48BDE1703E2A6719:C0E9DEAA90D60AE1]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1591)
	at org.apache.solr.update.TestInPlaceUpdatesDistrib.docValuesUpdateTest(TestInPlaceUpdatesDistrib.java:353)
	at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 54 more




Build Log:
[...truncated 9 lines...]
ERROR: Error cloning remote repo 'origin'
hudson.plugins.git.GitException: Command "git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*" returned status code 128:
stdout: 
stderr: remote: Counting objects: 56081           
remote: Counting objects: 77530           
remote: Counting objects: 131469           
remote: Counting objects: 224854           
remote: Counting objects: 329941           
remote: Counting objects: 411757           
remote: Counting objects: 486512           
remote: Counting objects: 562357           
remote: Counting objects: 639671           
remote: Counting objects: 731730           
remote: Counting objects: 821842           
remote: Counting objects: 919645           
remote: Counting objects: 975543, done.        
remote: Compressing objects:   0% (1/191820)           
remote: Compressing objects:   1% (1919/191820)           
remote: Compressing objects:   2% (3837/191820)           
remote: Compressing objects:   3% (5755/191820)           
remote: Compressing objects:   4% (7673/191820)           
remote: Compressing objects:   5% (9591/191820)           
remote: Compressing objects:   6% (11510/191820)           
remote: Compressing objects:   7% (13428/191820)           
remote: Compressing objects:   8% (15346/191820)           
remote: Compressing objects:   9% (17264/191820)           
remote: Compressing objects:  10% (19182/191820)           
remote: Compressing objects:  11% (21101/191820)           
remote: Compressing objects:  12% (23019/191820)           
remote: Compressing objects:  13% (24937/191820)           
remote: Compressing objects:  14% (26855/191820)           
remote: Compressing objects:  15% (28773/191820)           
remote: Compressing objects:  16% (30692/191820)           
remote: Compressing objects:  17% (32610/191820)           
remote: Compressing objects:  18% (34528/191820)           
remote: Compressing objects:  19% (36446/191820)           
remote: Compressing objects:  20% (38364/191820)           
remote: Compressing objects:  21% (40283/191820)           
remote: Compressing objects:  22% (42201/191820)           
remote: Compressing objects:  23% (44119/191820)           
remote: Compressing objects:  24% (46037/191820)           
remote: Compressing objects:  25% (47955/191820)           
remote: Compressing objects:  26% (49874/191820)           
remote: Compressing objects:  27% (51792/191820)           
remote: Compressing objects:  28% (53710/191820)           
remote: Compressing objects:  29% (55628/191820)           
remote: Compressing objects:  30% (57546/191820)           
remote: Compressing objects:  31% (59465/191820)           
remote: Compressing objects:  32% (61383/191820)           
remote: Compressing objects:  33% (63301/191820)           
remote: Compressing objects:  34% (65219/191820)           
remote: Compressing objects:  35% (67137/191820)           
remote: Compressing objects:  36% (69056/191820)           
remote: Compressing objects:  37% (70974/191820)           
remote: Compressing objects:  38% (72892/191820)           
remote: Compressing objects:  39% (74810/191820)           
remote: Compressing objects:  40% (76728/191820)           
remote: Compressing objects:  41% (78647/191820)           
remote: Compressing objects:  42% (80565/191820)           
remote: Compressing objects:  43% (82483/191820)           
remote: Compressing objects:  44% (84401/191820)           
remote: Compressing objects:  45% (86319/191820)           
remote: Compressing objects:  46% (88238/191820)           
remote: Compressing objects:  47% (90156/191820)           
remote: Compressing objects:  48% (92074/191820)           
remote: Compressing objects:  49% (93992/191820)           
remote: Compressing objects:  50% (95910/191820)           
remote: Compressing objects:  51% (97829/191820)           
remote: Compressing objects:  52% (99747/191820)           
remote: Compressing objects:  53% (101665/191820)           
remote: Compressing objects:  54% (103583/191820)           
remote: Compressing objects:  55% (105501/191820)           
remote: Compressing objects:  56% (107420/191820)           
remote: Compressing objects:  57% (109338/191820)           
remote: Compressing objects:  58% (111256/191820)           
remote: Compressing objects:  59% (113174/191820)           
remote: Compressing objects:  60% (115092/191820)           
remote: Compressing objects:  61% (117011/191820)           
remote: Compressing objects:  62% (118929/191820)           
remote: Compressing objects:  63% (120847/191820)           
remote: Compressing objects:  64% (122765/191820)           
remote: Compressing objects:  65% (124683/191820)           
remote: Compressing objects:  66% (126602/191820)           
remote: Compressing objects:  67% (128520/191820)           
remote: Compressing objects:  68% (130438/191820)           
remote: Compressing objects:  69% (132356/191820)           
remote: Compressing objects:  70% (134274/191820)           
remote: Compressing objects:  71% (136193/191820)           
remote: Compressing objects:  72% (138111/191820)           
remote: Compressing objects:  73% (140029/191820)           
remote: Compressing objects:  74% (141947/191820)           
remote: Compressing objects:  75% (143865/191820)           
remote: Compressing objects:  76% (145784/191820)           
remote: Compressing objects:  77% (147702/191820)           
remote: Compressing objects:  78% (149620/191820)           
remote: Compressing objects:  79% (151538/191820)           
remote: Compressing objects:  80% (153456/191820)           
remote: Compressing objects:  81% (155375/191820)           
remote: Compressing objects:  82% (157293/191820)           
remote: Compressing objects:  83% (159211/191820)           
remote: Compressing objects:  84% (161129/191820)           
remote: Compressing objects:  85% (163047/191820)           
remote: Compressing objects:  85% (164913/191820)           
remote: Compressing objects:  86% (164966/191820)           
remote: Compressing objects:  87% (166884/191820)           
remote: Compressing objects:  88% (168802/191820)           
remote: Compressing objects:  89% (170720/191820)           
remote: Compressing objects:  90% (172638/191820)           
remote: Compressing objects:  91% (174557/191820)           
remote: Compressing objects:  92% (176475/191820)           
remote: Compressing objects:  93% (178393/191820)           
remote: Compressing objects:  93% (179915/191820)           
remote: Compressing objects:  94% (180311/191820)           
remote: Compressing objects:  94% (181818/191820)           
remote: Compressing objects:  95% (182229/191820)           
remote: Compressing objects:  96% (184148/191820)           
remote: Compressing objects:  96% (184571/191820)           
remote: Compressing objects:  97% (186066/191820)           
fatal: The remote end hung up unexpectedly
fatal: protocol error: bad pack header

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandWithCredentials(CliGitAPIImpl.java:1721)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.access$300(CliGitAPIImpl.java:72)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl$1.execute(CliGitAPIImpl.java:405)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl$2.execute(CliGitAPIImpl.java:614)
	at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:153)
	at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:146)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.execute(RemoteGitImpl.java:146)
		at sun.reflect.GeneratedMethodAccessor905.invoke(Unknown Source)
		at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
		at java.lang.reflect.Method.invoke(Method.java:498)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.invoke(RemoteGitImpl.java:132)
		at com.sun.proxy.$Proxy130.execute(Unknown Source)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1146)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
		at hudson.scm.SCM.checkout(SCM.java:504)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1794)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error cloning remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
Fetching upstream changes from git://git.apache.org/lucene-solr.git
 > git --version # timeout=10
 > git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
 > git rev-parse refs/remotes/origin/master^{commit} # timeout=10
 > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10
Checking out Revision 0545901c24f4d146f4251ba52defa4eb23fc10b9 (refs/remotes/origin/master)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f 0545901c24f4d146f4251ba52defa4eb23fc10b9
Commit message: "Tighten up the test variability to reduce the risk of some random collection/query combinations that can't be verified"
 > git rev-list --no-walk 26f76cc1be0a5164142a64b588df4d80bcf8d245 # timeout=10
Cleaning workspace
 > git rev-parse --verify HEAD # timeout=10
Resetting working tree
 > git reset --hard # timeout=10
 > git clean -fdx # timeout=10
Checking out a fresh workspace because /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data doesn't exist
Cleaning local Directory test-data
Checking out http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-07-13T07:49:41.759 +0000'
A         README.txt
A         enwiki.random.lines.txt
At revision 1835789

No emails were triggered.
[checkout] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant -file build.xml -Dtests.multiplier=2 -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt jenkins-nightly
Buildfile: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml

jenkins-nightly:

-print-java-info:
[java-info] java version "1.8.0_172"
[java-info] Java(TM) SE Runtime Environment (1.8.0_172-b11, Oracle Corporation)
[java-info] Java HotSpot(TM) 64-Bit Server VM (25.172-b11, Oracle Corporation)
[java-info] Test args: []

clean:

clean:

clean:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: Apache Ivy 2.4.0 - 20141213170938 :: http://ant.apache.org/ivy/ ::
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve-groovy:
[ivy:cachepath] :: resolving dependencies :: org.codehaus.groovy#groovy-all-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.codehaus.groovy#groovy-all;2.4.15 in public
[ivy:cachepath] :: resolution report :: resolve 184ms :: artifacts dl 3ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   1   |   0   |   0   |   0   ||   1   |   0   |
	---------------------------------------------------------------------

-test-with-heapdumps-enabled:
     [echo] Java HotSpot(TM) 64-Bit Server VM: Enabling heap dumps on OutOfMemoryError to dir '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps'.
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve-groovy:

test:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

-clover.load:

resolve-groovy:

-init-totals:

test-core:

-clover.disable:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

-clover.load:

-clover.classpath:

-clover.setup:

clover:

-check-git-state:

-git-cleanroot:

-copy-git-state:
     [copy] Copying 1 file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build

git-autoclean:

resolve:

init:

compile-core:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java
    [javac] Compiling 846 source files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/util/graph/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/document/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/util/bkd/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/compressing/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/lucene60/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/util/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/index/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/util/fst/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/analysis/tokenattributes/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/search/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/util/packed/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/util/mutable/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/lucene70/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/perfield/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/analysis/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/analysis/standard/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/store/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/blocktree/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/search/spans/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/search/similarities/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/util/automaton/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/codecs/lucene50/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java/org/apache/lucene/geo/package-info.class
     [copy] Copying 3 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java

compile-test-framework:

-check-git-state:

-git-cleanroot:

-copy-git-state:

git-autoclean:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

init:

compile-lucene-core:

-check-git-state:

-git-cleanroot:

-copy-git-state:

git-autoclean:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

init:

-clover.disable:

-clover.load:

-clover.classpath:

-clover.setup:

clover:

compile-core:

compile-codecs:

-check-git-state:

-git-cleanroot:

-copy-git-state:

git-autoclean:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

common.init:

compile-lucene-core:

init:

-clover.disable:

-clover.load:

-clover.classpath:

-clover.setup:

clover:

compile-core:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java
    [javac] Compiling 63 source files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/bloom/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/memory/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/simpletext/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/blockterms/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java/org/apache/lucene/codecs/blocktreeords/package-info.class
     [copy] Copying 3 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java

-clover.disable:

-clover.load:

-clover.classpath:

-clover.setup:

clover:

common.compile-core:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java
    [javac] Compiling 194 source files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/asserting/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/ramonly/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/mockfile/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/cheapbastard/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/mockrandom/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/cranky/package-info.class
    [javac] Creating empty /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java/org/apache/lucene/codecs/compressing/dummy/package-info.class
     [copy] Copying 4 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java

compile-core:

compile-test:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test
    [javac] Compiling 507 source files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test
    [javac] /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java:2663: warning: [try] auto-closeable resource closeable is never referenced in body of corresponding try statement
    [javac]          Closeable closeable = () -> IOUtils.close(toClose)) {
    [javac]                    ^
    [javac] /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java:53: warning: [cast] redundant cast to long
    [javac]       long weightedByteSize = (long) Math.round(liveRatio * byteSize);
    [javac]                               ^
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.
    [javac] 2 warnings
     [copy] Copying 3 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test

-mrjar-classes-uptodate:

resolve-groovy:

patch-mrjar-classes:
[patch-cls] Remapped: org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl
[patch-cls] Remapped: org/apache/lucene/codecs/compressing/LZ4
[patch-cls] Remapped: org/apache/lucene/document/BinaryPoint$2
[patch-cls] Remapped: org/apache/lucene/document/DoubleRange
[patch-cls] Remapped: org/apache/lucene/document/FloatRange
[patch-cls] Remapped: org/apache/lucene/document/IntRange
[patch-cls] Remapped: org/apache/lucene/document/LongRange
[patch-cls] Remapped: org/apache/lucene/index/BitsSlice
[patch-cls] Remapped: org/apache/lucene/index/CodecReader
[patch-cls] Remapped: org/apache/lucene/index/MergeReaderWrapper
[patch-cls] Remapped: org/apache/lucene/search/BooleanScorer$TailPriorityQueue
[patch-cls] Remapped: org/apache/lucene/util/BytesRef
[patch-cls] Remapped: org/apache/lucene/util/BytesRefArray
[patch-cls] Remapped: org/apache/lucene/util/CharsRef$UTF16SortedAsUTF8Comparator
[patch-cls] Remapped: org/apache/lucene/util/CharsRef
[patch-cls] Remapped: org/apache/lucene/util/IntsRef
[patch-cls] Remapped: org/apache/lucene/util/LongsRef
[patch-cls] Remapped: org/apache/lucene/util/StringHelper
[patch-cls] Remapped: org/apache/lucene/util/automaton/Automaton$Builder
[patch-cls] Remapped: org/apache/lucene/util/automaton/Automaton
[patch-cls] Remapped 20 class files for Java 9 to: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java9
    [touch] Creating /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/patch-mrjar.stamp

install-junit4-taskdef:

validate:

-init-totals:

-test:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/.caches/test-stats/core
   [junit4] <JUnit4> says hi! Master seed: 27679A7851D9DC89
   [junit4] Executing 497 suites with 3 JVMs.
   [junit4] 
   [junit4] Started J0 PID(8561@localhost).
   [junit4] Started J1 PID(8560@localhost).
   [junit4] Started J2 PID(8562@localhost).
   [junit4] Suite: org.apache.lucene.document.TestFieldType
   [junit4] Completed [1/497] on J1 in 1.21s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.document.TestField
   [junit4] Completed [2/497] on J0 in 3.56s, 23 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestOrdinalMap
   [junit4] Completed [3/497] on J2 in 5.79s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexManyDocuments
   [junit4] Completed [4/497] on J0 in 6.45s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestAutomatonQueryUnicode
   [junit4] Completed [5/497] on J0 in 0.40s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.document.TestDocument
   [junit4] Completed [6/497] on J0 in 0.98s, 12 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestManyFields
   [junit4] Completed [7/497] on J2 in 9.00s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestReqExclBulkScorer
   [junit4] Completed [8/497] on J2 in 0.31s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexOptions
   [junit4] Completed [9/497] on J1 in 13.90s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestQueryBuilder
   [junit4] Completed [10/497] on J2 in 0.32s, 23 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestBlendedTermQuery
   [junit4] Completed [11/497] on J1 in 0.38s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.document.TestDoubleRange
   [junit4] Completed [12/497] on J1 in 0.09s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSortedNumericSortField
   [junit4] Completed [13/497] on J2 in 1.25s, 9 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.compressing.TestHighCompressionMode
   [junit4] Completed [14/497] on J0 in 8.58s, 7 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.TestAssertions
   [junit4] Completed [15/497] on J0 in 0.19s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestTermdocPerf
   [junit4] Completed [16/497] on J0 in 0.05s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestDocCount
   [junit4] Completed [17/497] on J0 in 2.73s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.automaton.TestRegExp
   [junit4] Completed [18/497] on J0 in 0.64s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestMixedCodecs
   [junit4] Completed [19/497] on J0 in 14.89s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestOmitTf
   [junit4] Completed [20/497] on J0 in 1.42s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSearchWithThreads
   [junit4] Completed [21/497] on J2 in 36.19s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestScoreCachingWrappingScorer
   [junit4] Completed [22/497] on J2 in 0.19s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSimpleExplanations
   [junit4] Completed [23/497] on J2 in 9.68s, 70 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterLockRelease
   [junit4] Completed [24/497] on J2 in 0.09s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanMultiTermQueryWrapper
   [junit4] Completed [25/497] on J2 in 1.29s, 9 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestLSBRadixSorter
   [junit4] Completed [26/497] on J2 in 0.67s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestMatchAllDocsQuery
   [junit4] Completed [27/497] on J2 in 0.32s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BNumericDocValues
   [junit4] IGNOR/A 0.02s J2 | Test2BNumericDocValues.testNumerics
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes ~ 2 hours if the heap is 5gb))
   [junit4] Completed [28/497] on J2 in 0.02s, 1 test, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestHugeRamFile
   [junit4] Completed [29/497] on J2 in 6.53s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestStringMSBRadixSorter
   [junit4] Completed [30/497] on J2 in 14.71s, 7 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.automaton.TestOperations
   [junit4] Completed [31/497] on J2 in 11.91s, 7 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestParallelReaderEmptyIndex
   [junit4] Completed [32/497] on J2 in 0.11s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestTermRangeQuery
   [junit4] Completed [33/497] on J2 in 0.49s, 7 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestTermStates
   [junit4] Completed [34/497] on J2 in 0.19s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestLazyProxSkipping
   [junit4] Completed [35/497] on J2 in 1.97s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.TestSearchForDuplicates
   [junit4] Completed [36/497] on J2 in 0.63s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T07:58:13, stalled for 69.8s at: TestBlockPostingsFormat.testDocsAndFreqsAndPositionsAndOffsetsAndPayloads
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T07:58:37, stalled for 63.6s at: TestNumericDocValuesUpdates.testTonsOfUpdates
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T07:59:37, stalled for  124s at: TestNumericDocValuesUpdates.testTonsOfUpdates
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T07:59:37, stalled for 66.0s at: TestTermsEnum.testVaryingTermsPerSegment
   [junit4] Suite: org.apache.lucene.index.TestNumericDocValuesUpdates
   [junit4] Completed [37/497] on J0 in 175.80s, 32 tests
   [junit4] 
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:00:37, stalled for  126s at: TestTermsEnum.testVaryingTermsPerSegment
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat
   [junit4] Completed [38/497] on J1 in 267.26s, 28 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.TestDemo
   [junit4] Completed [39/497] on J1 in 0.17s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestTerms
   [junit4] Completed [40/497] on J1 in 0.98s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestDocIDMerger
   [junit4] Completed [41/497] on J1 in 0.06s, 2 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:01:01, stalled for 65.3s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.util.automaton.TestUTF32ToUTF8
   [junit4] Completed [42/497] on J1 in 16.85s, 6 tests
   [junit4] 
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:01:37, stalled for  186s at: TestTermsEnum.testVaryingTermsPerSegment
   [junit4] Suite: org.apache.lucene.index.TestTermsEnum
   [junit4] Completed [43/497] on J2 in 235.50s, 13 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:02:01, stalled for  125s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.util.TestOfflineSorter
   [junit4] Completed [44/497] on J2 in 41.01s, 15 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexInput
   [junit4] Completed [45/497] on J2 in 2.15s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestSparseFixedBitDocIdSet
   [junit4] Completed [46/497] on J2 in 0.67s, 5 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSort
   [junit4] Completed [47/497] on J2 in 0.49s, 25 tests
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:02:25, stalled for 66.3s at: TestMixedDocValuesUpdates.testTonsOfUpdates
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterMerging
   [junit4] Completed [48/497] on J2 in 11.98s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestRollingBuffer
   [junit4] Completed [49/497] on J2 in 0.79s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestBlockMaxConjunction
   [junit4] Completed [50/497] on J2 in 3.25s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestCharsRef
   [junit4] Completed [51/497] on J2 in 0.16s, 6 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:03:01, stalled for  186s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestMixedDocValuesUpdates
   [junit4] Completed [52/497] on J1 in 122.06s, 10 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestOneMergeWrappingMergePolicy
   [junit4] Completed [53/497] on J1 in 0.04s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestOmitNorms
   [junit4] Completed [54/497] on J1 in 4.58s, 5 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestTieredMergePolicy
   [junit4] Completed [55/497] on J2 in 45.21s, 11 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOutOfFileDescriptors
   [junit4] Completed [56/497] on J1 in 18.81s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.automaton.TestAutomaton
   [junit4] Completed [57/497] on J2 in 13.65s, 56 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestSegmentTermDocs
   [junit4] Completed [58/497] on J2 in 1.16s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestPointValues
   [junit4] Completed [59/497] on J1 in 22.49s, 32 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestMmapDirectory
   [junit4] Completed [60/497] on J1 in 5.38s, 43 tests, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestRamUsageEstimator
   [junit4] IGNOR/A 0.03s J1 | TestRamUsageEstimator.testPrintValues
   [junit4]    > Assumption #1: Specify -Dtests.verbose=true to print constants of RamUsageEstimator.
   [junit4] Completed [61/497] on J1 in 0.11s, 5 tests, 1 skipped
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:04:01, stalled for  246s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestLucene50CompoundFormat
   [junit4] Completed [62/497] on J2 in 21.21s, 26 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BSortedDocValuesOrds
   [junit4] IGNOR/A 0.00s J2 | Test2BSortedDocValuesOrds.test2BOrds
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=Takes ~ 6 hours if the heap is 5gb))
   [junit4] Completed [63/497] on J2 in 0.01s, 1 test, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestCharArraySet
   [junit4] Completed [64/497] on J2 in 0.15s, 15 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.document.TestFeatureField
   [junit4] Completed [65/497] on J2 in 0.67s, 8 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanBoostQuery
   [junit4] Completed [66/497] on J2 in 0.04s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestArrayUtil
   [junit4] Completed [67/497] on J2 in 10.59s, 15 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestDoubleValuesSource
   [junit4] Completed [68/497] on J2 in 24.60s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestMultiTermsEnum
   [junit4] Completed [69/497] on J2 in 0.05s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:05:02, stalled for  306s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:05:02, stalled for 63.9s at: TestLMJelinekMercerSimilarity.testRandomScoring
   [junit4] Suite: org.apache.lucene.search.similarities.TestLMJelinekMercerSimilarity
   [junit4] Completed [70/497] on J1 in 77.24s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestTerm
   [junit4] Completed [71/497] on J1 in 0.03s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestCustomSearcherSort
   [junit4] Completed [72/497] on J1 in 8.69s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestComplexExplanations
   [junit4] Completed [73/497] on J1 in 3.27s, 21 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSimilarityProvider
   [junit4] Completed [74/497] on J1 in 0.06s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestPayloads
   [junit4] Completed [75/497] on J1 in 0.52s, 7 tests
   [junit4] 
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:05:50, stalled for 71.5s at: TestTopFieldCollectorEarlyTermination.testEarlyTermination
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:06:02, stalled for  366s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:06:38, stalled for 63.4s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] Suite: org.apache.lucene.search.TestTopFieldCollectorEarlyTermination
   [junit4] Completed [76/497] on J2 in 140.33s, 3 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:07:02, stalled for  426s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:07:38, stalled for  123s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:08:02, stalled for 63.3s at: TestBasicModelIF.testRandomScoring
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:08:02, stalled for  486s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:08:38, stalled for  184s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] Suite: org.apache.lucene.search.similarities.TestBasicModelIF
   [junit4] Completed [77/497] on J2 in 110.24s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestExitableDirectoryReader
   [junit4] Completed [78/497] on J2 in 0.03s, 1 test, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestComplexExplanationsOfNonMatches
   [junit4] Completed [79/497] on J2 in 0.70s, 21 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestMergeRateLimiter
   [junit4] Completed [80/497] on J2 in 0.29s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:09:02, stalled for  546s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestIndexWriter
   [junit4] Completed [81/497] on J2 in 25.77s, 87 tests, 1 skipped
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:09:38, stalled for  244s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:10:02, stalled for  606s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.codecs.lucene60.TestLucene60PointsFormat
   [junit4] IGNOR/A 0.00s J2 | TestLucene60PointsFormat.testMergeStability
   [junit4]    > Assumption #1: merge is not stable
   [junit4] Completed [82/497] on J2 in 68.14s, 20 tests, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.MultiCollectorTest
   [junit4] Completed [83/497] on J2 in 0.05s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestDateSort
   [junit4] Completed [84/497] on J2 in 0.07s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.geo.TestGeoEncodingUtils
   [junit4] Completed [85/497] on J2 in 2.53s, 3 tests
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:10:38, stalled for  304s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:11:03, stalled for  667s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.search.similarities.TestAxiomaticF1EXP
   [junit4] Completed [86/497] on J2 in 47.47s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestFieldCacheRewriteMethod
   [junit4] Completed [87/497] on J2 in 15.21s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestSumDocFreq
   [junit4] Completed [88/497] on J2 in 1.31s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestPositionIncrement
   [junit4] Completed [89/497] on J2 in 0.20s, 2 tests
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:11:39, stalled for  364s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterWithThreads
   [junit4] Completed [90/497] on J2 in 13.25s, 14 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestFileSwitchDirectory
   [junit4] IGNOR/A 0.02s J2 | TestFileSwitchDirectory.testPendingDeletions
   [junit4]    > Assumption #1: we can only install VirusCheckingFS on an FSDirectory
   [junit4] IGNOR/A 0.01s J2 | TestFileSwitchDirectory.testFsyncDoesntCreateNewFiles
   [junit4]    > Assumption #1: test only works for FSDirectory subclasses
   [junit4] Completed [91/497] on J2 in 4.58s, 43 tests, 2 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.automaton.TestDeterminizeLexicon
   [junit4] Completed [92/497] on J2 in 6.86s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:12:03, stalled for  727s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnJRECrash
   [junit4] Completed [93/497] on J2 in 11.82s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:12:39, stalled for  424s at: TestIndexWriterExceptions.testTooManyTokens
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:13:03, stalled for  787s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestDuelingCodecs
   [junit4] Completed [94/497] on J2 in 74.98s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestDocumentWriter
   [junit4] Completed [95/497] on J2 in 0.51s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterDelete
   [junit4] Completed [96/497] on J2 in 32.23s, 26 tests, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestBytesRef
   [junit4] Completed [97/497] on J2 in 0.15s, 4 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:14:03, stalled for  847s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterExceptions
   [junit4] Completed [98/497] on J1 in 524.09s, 35 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestIntArrayDocIdSet
   [junit4] Completed [99/497] on J1 in 0.56s, 5 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestFilterMergePolicy
   [junit4] Completed [100/497] on J1 in 0.01s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestMaxTermFrequency
   [junit4] Completed [101/497] on J1 in 0.25s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestMultiMMap
   [junit4] Completed [102/497] on J2 in 22.78s, 54 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestPayloadsOnVectors
   [junit4] Completed [103/497] on J2 in 0.15s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestReaderClosed
   [junit4] Completed [104/497] on J2 in 0.41s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterUnicode
   [junit4] Completed [105/497] on J1 in 10.58s, 5 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestMSBRadixSorter
   [junit4] Completed [106/497] on J2 in 7.82s, 8 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.bkd.Test2BBKDPoints
   [junit4] IGNOR/A 0.00s J2 | Test2BBKDPoints.test1D
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes at least 4 hours and consumes many GB of temp disk space))
   [junit4] IGNOR/A 0.00s J2 | Test2BBKDPoints.test2D
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes at least 4 hours and consumes many GB of temp disk space))
   [junit4] Completed [107/497] on J2 in 0.02s, 2 tests, 2 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestNamedSPILoader
   [junit4] Completed [108/497] on J2 in 0.03s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestPostingsOffsets
   [junit4] Completed [109/497] on J2 in 6.44s, 13 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestFieldReuse
   [junit4] Completed [110/497] on J2 in 0.03s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.similarities.TestAxiomaticSimilarity
   [junit4] Completed [111/497] on J2 in 0.03s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanExplanations
   [junit4] Completed [112/497] on J2 in 2.40s, 30 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestOmitPositions
   [junit4] Completed [113/497] on J2 in 0.33s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.spans.TestFieldMaskingSpanQuery
   [junit4] IGNOR/A 0.00s J2 | TestFieldMaskingSpanQuery.testSpans2
   [junit4]    > Assumption #1: Broken scoring: LUCENE-3723
   [junit4] IGNOR/A 0.00s J2 | TestFieldMaskingSpanQuery.testSimple2
   [junit4]    > Assumption #1: Broken scoring: LUCENE-3723
   [junit4] Completed [114/497] on J2 in 0.35s, 11 tests, 2 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestForceMergeForever
   [junit4] Completed [115/497] on J2 in 1.48s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestFrequencyTrackingRingBuffer
   [junit4] Completed [116/497] on J2 in 0.94s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanOrQuery
   [junit4] Completed [117/497] on J2 in 0.04s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.document.TestFloatRange
   [junit4] Completed [118/497] on J2 in 0.04s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestAllFilesDetectTruncation
   [junit4] Completed [119/497] on J1 in 18.95s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestSmallFloat
   [junit4] Completed [120/497] on J1 in 0.62s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestWildcardRandom
   [junit4] Completed [121/497] on J1 in 0.62s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestMatchesIterator
   [junit4] Completed [122/497] on J1 in 0.93s, 23 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:15:03, stalled for  907s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.store.TestGrowableByteArrayDataOutput
   [junit4] Completed [123/497] on J1 in 29.94s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestBoolean2ScorerSupplier
   [junit4] Completed [124/497] on J1 in 0.68s, 10 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestTermQuery
   [junit4] Completed [125/497] on J1 in 0.06s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.fst.TestBytesStore
   [junit4] Completed [126/497] on J1 in 21.02s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.similarities.TestIndependenceStandardized
   [junit4] Completed [127/497] on J2 in 58.19s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.automaton.TestMinimize
   [junit4] Completed [128/497] on J1 in 10.04s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestUsageTrackingFilterCachingPolicy
   [junit4] Completed [129/497] on J1 in 0.10s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestTopFieldCollector
   [junit4] Completed [130/497] on J1 in 0.92s, 8 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestNativeFSLockFactory
   [junit4] Completed [131/497] on J2 in 16.22s, 11 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestCodecHoldsOpenFiles
   [junit4] Completed [132/497] on J2 in 0.14s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:16:03, stalled for  967s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.search.TestSimpleSearchEquivalence
   [junit4] Completed [133/497] on J2 in 18.01s, 17 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.TestExternalCodecs
   [junit4] Completed [134/497] on J2 in 0.98s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.TestMergeSchedulerExternal
   [junit4] Completed [135/497] on J2 in 1.29s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.TestSearch
   [junit4] Completed [136/497] on J2 in 0.27s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestCachingTokenFilter
   [junit4] Completed [137/497] on J2 in 0.02s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestCharArrayMap
   [junit4] Completed [138/497] on J2 in 0.25s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestDelegatingAnalyzerWrapper
   [junit4] Completed [139/497] on J2 in 0.12s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestGraphTokenizers
   [junit4] Completed [140/497] on J2 in 5.40s, 23 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestReusableStringReader
   [junit4] Completed [141/497] on J2 in 0.04s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestStopFilter
   [junit4] Completed [142/497] on J2 in 0.03s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestToken
   [junit4] Completed [143/497] on J2 in 0.05s, 5 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestWordlistLoader
   [junit4] Completed [144/497] on J2 in 0.03s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.standard.TestStandardAnalyzer
   [junit4] Completed [145/497] on J2 in 28.74s, 36 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.tokenattributes.TestBytesRefAttImpl
   [junit4] Completed [146/497] on J2 in 0.02s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:16:51, stalled for 64.0s at: TestIndexWriterOnDiskFull.testAddIndexOnDiskFull
   [junit4] Suite: org.apache.lucene.analysis.tokenattributes.TestCharTermAttributeImpl
   [junit4] Completed [147/497] on J2 in 0.98s, 13 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.tokenattributes.TestPackedTokenAttributeImpl
   [junit4] Completed [148/497] on J2 in 0.03s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.tokenattributes.TestSimpleAttributeImpl
   [junit4] Completed [149/497] on J2 in 0.04s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.TestCodecLoadingDeadlock
   [junit4] Completed [150/497] on J2 in 0.44s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.TestCodecUtil
   [junit4] Completed [151/497] on J2 in 0.03s, 16 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.TestCompetitiveFreqNormAccumulator
   [junit4] Completed [152/497] on J2 in 0.04s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.compressing.TestFastCompressionMode
   [junit4] Completed [153/497] on J2 in 2.16s, 11 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.compressing.TestFastDecompressionMode
   [junit4] Completed [154/497] on J2 in 5.06s, 11 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:17:03, stalled for 1027s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnDiskFull
   [junit4] Completed [155/497] on J1 in 78.81s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat2
   [junit4] Completed [156/497] on J2 in 7.75s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3
   [junit4] Completed [157/497] on J2 in 21.97s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestForUtil
   [junit4] Completed [158/497] on J2 in 0.07s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:18:03, stalled for 1087s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestLucene50StoredFieldsFormat
   [junit4] Completed [159/497] on J1 in 66.93s, 17 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestByteBlockPool
   [junit4] Completed [160/497] on J1 in 0.66s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestNRTReaderWithThreads
   [junit4] Completed [161/497] on J1 in 7.12s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.automaton.LimitedFiniteStringsIteratorTest
   [junit4] Completed [162/497] on J1 in 0.51s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSearchAfter
   [junit4] Completed [163/497] on J1 in 3.80s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSameScoresWithThreads
   [junit4] Completed [164/497] on J1 in 3.33s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSloppyPhraseQuery2
   [junit4] Completed [165/497] on J1 in 29.20s, 9 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestRateLimiter
   [junit4] Completed [166/497] on J1 in 0.62s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.document.TestLongRange
   [junit4] Completed [167/497] on J1 in 0.03s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestThreadedForceMerge
   [junit4] Completed [168/497] on J1 in 1.56s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.analysis.TestCharacterUtils
   [junit4] Completed [169/497] on J1 in 0.04s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSortedSetSortField
   [junit4] Completed [170/497] on J1 in 0.22s, 7 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestBooleanQuery
   [junit4] Completed [171/497] on J1 in 0.97s, 22 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.spans.TestNearSpansOrdered
   [junit4] Completed [172/497] on J1 in 0.71s, 19 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:19:03, stalled for 1147s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:19:03, stalled for 71.6s at: TestLucene50StoredFieldsFormatHighCompression.testBigDocuments
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestLucene50StoredFieldsFormatHighCompression
   [junit4] Completed [173/497] on J2 in 129.10s, 19 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:20:03, stalled for 1207s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:20:15, stalled for 66.0s at: TestIndexingSequenceNumbers.testStressConcurrentAddAndDeleteAndCommit
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:21:03, stalled for 1267s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:21:15, stalled for  126s at: TestIndexingSequenceNumbers.testStressConcurrentAddAndDeleteAndCommit
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:21:15, stalled for 62.8s at: TestLucene50TermVectorsFormat.testRamBytesUsed
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:22:03, stalled for 1328s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestLucene50TermVectorsFormat
   [junit4] Completed [174/497] on J2 in 150.40s, 17 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestLucene60FieldInfoFormat
   [junit4] Completed [175/497] on J2 in 0.80s, 11 tests
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:22:15, stalled for  186s at: TestIndexingSequenceNumbers.testStressConcurrentAddAndDeleteAndCommit
   [junit4] Suite: org.apache.lucene.codecs.lucene70.TestIndexedDISI
   [junit4] Completed [176/497] on J2 in 9.32s, 10 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:23:03, stalled for 1388s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:24:03, stalled for 1448s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.index.TestIndexingSequenceNumbers
   [junit4] Completed [177/497] on J1 in 302.06s, 8 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestDirectory
   [junit4] Completed [178/497] on J1 in 0.61s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.similarities.TestIndependenceSaturated
   [junit4] Completed [179/497] on J1 in 57.39s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterConfig
   [junit4] Completed [180/497] on J1 in 0.18s, 8 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestParallelLeafReader
   [junit4] Completed [181/497] on J1 in 0.47s, 10 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.mutable.TestMutableValues
   [junit4] Completed [182/497] on J1 in 0.05s, 6 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:25:03, stalled for 1508s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:25:27, stalled for 70.6s at: TestLucene70DocValuesFormat.testTermsEnumRandomMany
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:26:03, stalled for 1568s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.search.TestDoubleRangeFieldQueries
   [junit4] Completed [183/497] on J1 in 84.93s, 5 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexTooManyDocs
   [junit4] Completed [184/497] on J1 in 0.32s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestVirtualMethod
   [junit4] Completed [185/497] on J1 in 0.06s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestLongValuesSource
   [junit4] Completed [186/497] on J1 in 9.65s, 4 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:27:04, stalled for 1628s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.search.similarities.TestAxiomaticF3LOG
   [junit4] Completed [187/497] on J1 in 27.68s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestFieldInvertState
   [junit4] Completed [188/497] on J1 in 0.19s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestPrefixInBooleanQuery
   [junit4] Completed [189/497] on J1 in 6.49s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.Test2BPagedBytes
   [junit4] IGNOR/A 0.00s J1 | Test2BPagedBytes.test
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=You must increase heap to > 2 G to run this))
   [junit4] Completed [190/497] on J1 in 0.04s, 1 test, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestCloseableThreadLocal
   [junit4] Completed [191/497] on J1 in 0.06s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestCrashCausesCorruptIndex
   [junit4] Completed [192/497] on J1 in 0.35s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestBufferedChecksum
   [junit4] Completed [193/497] on J1 in 0.16s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.geo.TestPolygon
   [junit4] Completed [194/497] on J1 in 0.06s, 15 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestDocIdSetIterator
   [junit4] Completed [195/497] on J1 in 0.03s, 5 tests
   [junit4] 
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:27:28, stalled for 66.4s at: TestLucene70DocValuesFormat.testSparseDocValuesVsStoredFields
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:28:04, stalled for 1688s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:28:16, stalled for 62.8s at: TestDistributionLL.testRandomScoring
   [junit4] Suite: org.apache.lucene.search.similarities.TestDistributionLL
   [junit4] Completed [196/497] on J1 in 74.20s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestLongPostings
   [junit4] Completed [197/497] on J1 in 35.21s, 2 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:29:04, stalled for 1748s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] Suite: org.apache.lucene.search.TestQueryRescorer
   [junit4] Completed [198/497] on J1 in 3.62s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestFixedLengthBytesRefArray
   [junit4] Completed [199/497] on J1 in 0.28s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.spans.TestSpanFirstQuery
   [junit4] Completed [200/497] on J1 in 0.02s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestForTooMuchCloning
   [junit4] Completed [201/497] on J1 in 0.16s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:30:04, stalled for 1808s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:31:04, stalled for 1868s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:32:04, stalled for 1928s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:32:16, stalled for 61.1s at: TestLucene70DocValuesFormat.testSortedSetVariableLengthBigVsStoredFields
   [junit4] Suite: org.apache.lucene.codecs.perfield.TestPerFieldDocValuesFormat
   [junit4] Completed [202/497] on J1 in 196.36s, 110 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestLongsRef
   [junit4] Completed [203/497] on J1 in 0.03s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestMultiPhraseEnum
   [junit4] Completed [204/497] on J1 in 0.03s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestApproximationSearchEquivalence
   [junit4] Completed [205/497] on J1 in 26.30s, 10 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestTimeLimitingCollector
   [junit4] Completed [206/497] on J1 in 4.80s, 7 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:33:04, stalled for 1988s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:33:16, stalled for  121s at: TestLucene70DocValuesFormat.testSortedSetVariableLengthBigVsStoredFields
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:34:04, stalled for 2048s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:34:40, stalled for 69.3s at: TestLucene70DocValuesFormat.testSortedNumericBlocksOfVariousBitsPerValue
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:35:04, stalled for 2108s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:35:52, stalled for 61.3s at: TestLucene70NormsFormat.testNCommonBig
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:36:04, stalled for 2168s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:36:52, stalled for  121s at: TestLucene70NormsFormat.testNCommonBig
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:37:04, stalled for 2228s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:37:16, stalled for 69.6s at: TestLucene70DocValuesFormat.testTermsEnumVariableWidth
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:38:04, stalled for 2288s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:38:40, stalled for 69.7s at: TestLucene70DocValuesFormat.testSortedVariableLengthBigVsStoredFields
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:38:52, stalled for 69.1s at: TestLucene70NormsFormat.testSparseNCommonBig
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:39:04, stalled for 2348s at: TestIndexWriterOnVMError.testCheckpoint
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:39:52, stalled for  129s at: TestLucene70NormsFormat.testSparseNCommonBig
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnVMError
   [junit4] Completed [207/497] on J0 in 2425.95s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestPersistentSnapshotDeletionPolicy
   [junit4] Completed [208/497] on J0 in 2.10s, 13 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestLockFactory
   [junit4] Completed [209/497] on J0 in 0.06s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestRadixSelector
   [junit4] Completed [210/497] on J0 in 4.95s, 2 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.lucene70.TestLucene70NormsFormat
   [junit4] IGNOR/A 0.01s J1 | TestLucene70NormsFormat.testMergeStability
   [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
   [junit4] Completed [211/497] on J1 in 451.84s, 29 tests, 1 skipped
   [junit4] 
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:40:52, stalled for 67.7s at: TestLucene70DocValuesFormat.testTermsEnumLongSharedPrefixes
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:41:16, stalled for 64.2s at: TestLMDirichletSimilarity.testRandomScoring
   [junit4] Suite: org.apache.lucene.search.similarities.TestLMDirichletSimilarity
   [junit4] Completed [212/497] on J0 in 85.93s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestSwappedIndexFiles
   [junit4] Completed [213/497] on J0 in 0.47s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:41:41, stalled for 71.6s at: TestPerFieldPostingsFormat.testDocsAndFreqsAndPositionsAndOffsetsAndPayloads
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:41:53, stalled for  128s at: TestLucene70DocValuesFormat.testTermsEnumLongSharedPrefixes
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:42:40, stalled for 61.9s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:42:53, stalled for 69.3s at: TestPerFieldPostingsFormat.testDocsAndFreqsAndPositionsAndPayloads
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:42:53, stalled for  188s at: TestLucene70DocValuesFormat.testTermsEnumLongSharedPrefixes
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:43:40, stalled for  122s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:43:53, stalled for  248s at: TestLucene70DocValuesFormat.testTermsEnumLongSharedPrefixes
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:44:40, stalled for  182s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] Suite: org.apache.lucene.codecs.perfield.TestPerFieldPostingsFormat
   [junit4] IGNOR/A 0.02s J1 | TestPerFieldPostingsFormat.testPostingsEnumReuse
   [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
   [junit4] IGNOR/A 0.00s J1 | TestPerFieldPostingsFormat.testMergeStability
   [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
   [junit4] Completed [214/497] on J1 in 264.77s, 25 tests, 2 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.perfield.TestPerFieldPostingsFormat2
   [junit4] Completed [215/497] on J1 in 18.95s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.document.TestBinaryDocument
   [junit4] Completed [216/497] on J1 in 0.16s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.document.TestDateTools
   [junit4] Completed [217/497] on J1 in 0.09s, 5 tests
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:45:40, stalled for  242s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] Suite: org.apache.lucene.geo.TestPolygon2D
   [junit4] Completed [218/497] on J1 in 34.47s, 10 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BBinaryDocValues
   [junit4] IGNOR/A 0.00s J1 | Test2BBinaryDocValues.testFixedBinary
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes ~ 6 hours if the heap is 5gb))
   [junit4] IGNOR/A 0.00s J1 | Test2BBinaryDocValues.testVariableBinary
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes ~ 6 hours if the heap is 5gb))
   [junit4] Completed [219/497] on J1 in 0.02s, 2 tests, 2 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BDocs
   [junit4] IGNOR/A 0.00s J1 | Test2BDocs.test2BDocs
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=Takes ~30min))
   [junit4] Completed [220/497] on J1 in 0.03s, 1 test, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BPoints
   [junit4] IGNOR/A 0.00s J1 | Test2BPoints.test1D
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes at least 4 hours and consumes many GB of temp disk space))
   [junit4] IGNOR/A 0.00s J1 | Test2BPoints.test2D
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes at least 4 hours and consumes many GB of temp disk space))
   [junit4] Completed [221/497] on J1 in 0.01s, 2 tests, 2 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BPositions
   [junit4] IGNOR/A 0.00s J1 | Test2BPositions.test
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=uses lots of space and takes a few minutes))
   [junit4] Completed [222/497] on J1 in 0.01s, 1 test, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.codecs.lucene70.TestLucene70DocValuesFormat
   [junit4] Completed [223/497] on J2 in 1450.73s, 126 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BPostingsBytes
   [junit4] IGNOR/A 0.00s J2 | Test2BPostingsBytes.test
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=takes ~20GB-30GB of space and 10 minutes))
   [junit4] Completed [224/497] on J2 in 0.02s, 1 test, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BSortedDocValuesFixedSorted
   [junit4] IGNOR/A 0.00s J2 | Test2BSortedDocValuesFixedSorted.testFixedSorted
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=Takes ~ 6 hours if the heap is 5gb))
   [junit4] Completed [225/497] on J2 in 0.03s, 1 test, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.Test2BTerms
   [junit4] IGNOR/A 0.00s J2 | Test2BTerms.test2BTerms
   [junit4]    > Assumption #1: 'monster' test group is disabled (@Monster(value=very slow, use 5g minimum heap))
   [junit4] Completed [226/497] on J2 in 0.01s, 1 test, 1 skipped
   [junit4] 
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:46:40, stalled for  302s at: TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:46:53, stalled for 67.0s at: Test2BPostings.test
   [junit4] Suite: org.apache.lucene.search.TestControlledRealTimeReopenThread
   [junit4] IGNOR/A 0.00s J0 | TestControlledRealTimeReopenThread.testCRTReopen
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/LUCENE-5737))
   [junit4] Completed [227/497] on J0 in 330.98s, 6 tests, 1 skipped
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestCachingCollector
   [junit4] Completed [228/497] on J0 in 0.08s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestSimilarity
   [junit4] Completed [229/497] on J0 in 0.10s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestParallelTermEnum
   [junit4] Completed [230/497] on J0 in 0.05s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterDeleteByQuery
   [junit4] Completed [231/497] on J0 in 0.09s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestInPlaceMergeSorter
   [junit4] Completed [232/497] on J0 in 0.34s, 9 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestFilterLeafReader
   [junit4] Completed [233/497] on J0 in 0.07s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.index.TestMaxPosition
   [junit4] Completed [234/497] on J0 in 0.06s, 2 tests
   [junit4] 
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:47:41, stalled for 68.4s at: Test4GBStoredFields.test
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:47:53, stalled for  127s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:48:17, stalled for 64.1s at: TestIndexWriterThreadsToSegments.testSegmentCountOnFlushRandom
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:48:42, stalled for  128s at: Test4GBStoredFields.test
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:48:54, stalled for  187s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:49:17, stalled for  124s at: TestIndexWriterThreadsToSegments.testSegmentCountOnFlushRandom
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:49:42, stalled for  189s at: Test4GBStoredFields.test
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:49:54, stalled for  248s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:50:17, stalled for  184s at: TestIndexWriterThreadsToSegments.testSegmentCountOnFlushRandom
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:50:42, stalled for  249s at: Test4GBStoredFields.test
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:50:54, stalled for  308s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:51:17, stalled for  244s at: TestIndexWriterThreadsToSegments.testSegmentCountOnFlushRandom
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:51:42, stalled for  309s at: Test4GBStoredFields.test
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:51:54, stalled for  368s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:52:17, stalled for  304s at: TestIndexWriterThreadsToSegments.testSegmentCountOnFlushRandom
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:52:42, stalled for  369s at: Test4GBStoredFields.test
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:52:54, stalled for  428s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:53:17, stalled for  364s at: TestIndexWriterThreadsToSegments.testSegmentCountOnFlushRandom
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:53:42, stalled for  429s at: Test4GBStoredFields.test
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:53:54, stalled for  488s at: Test2BPostings.test
   [junit4] HEARTBEAT J0 PID(8561@localhost): 2018-07-13T08:54:17, stalled for  424s at: TestIndexWriterThreadsToSegments.testSegmentCountOnFlushRandom
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:54:42, stalled for  489s at: Test4GBStoredFields.test
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterThreadsToSegments
   [junit4] Completed [235/497] on J0 in 456.98s, 4 tests
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:54:54, stalled for  548s at: Test2BPostings.test
   [junit4] Suite: org.apache.lucene.index.TestAllFilesCheckIndexHeader
   [junit4] Completed [236/497] on J0 in 11.53s, 1 test
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.TestSloppyMath
   [junit4] Completed [237/497] on J0 in 2.49s, 8 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestWildcard
   [junit4] Completed [238/497] on J0 in 0.54s, 8 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestMatchNoDocsQuery
   [junit4] Completed [239/497] on J0 in 0.07s, 3 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.store.TestFilterDirectory
   [junit4] IGNOR/A 0.00s J0 | TestFilterDirectory.testFsyncDoesntCreateNewFiles
   [junit4]    > Assumption #1: test only works for FSDirectory subclasses
   [junit4] IGNOR/A 0.02s J0 | TestFilterDirectory.testPendingDeletions
   [junit4]    > Assumption #1: we can only install VirusCheckingFS on an FSDirectory
   [junit4] Completed [240/497] on J0 in 4.97s, 44 tests, 2 skipped
   [junit4] 
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:55:42, stalled for  549s at: Test4GBStoredFields.test
   [junit4] Suite: org.apache.lucene.search.TestBoolean2
   [junit4] Completed [241/497] on J0 in 38.50s, 10 tests
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:55:54, stalled for  608s at: Test2BPostings.test
   [junit4] Suite: org.apache.lucene.search.TestSynonymQuery
   [junit4] Completed [242/497] on J0 in 14.05s, 6 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.util.automaton.TestLevenshteinAutomata
   [junit4] Completed [244/497] on J0 in 4.19s, 4 tests
   [junit4] 
   [junit4] Suite: org.apache.lucene.search.TestTotalHitCountCollector
   [junit4] Completed [245/497] on J0 in 0.06s, 1 test
   [junit4] 
   [junit4] HEARTBEAT J2 PID(8562@localhost): 2018-07-13T08:56:42, stalled for  609s at: Test4GBStoredFields.test
   [junit4] Suite: org.apache.lucene.search.TestTopDocsMerge
   [junit4] Completed [246/497] on J0 in 45.96s, 4 tests
   [junit4] 
   [junit4] HEARTBEAT J1 PID(8560@localhost): 2018-07-13T08:56:54, stalled for  668s at: Test2BPostings.test
   [junit4] Suite

[...truncated too long message...]



ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null1458520113
     [copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null1458520113
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null1458520113

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 91ms :: artifacts dl 7ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 447 minutes 55 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$34.hasMatch(FilePath.java:2678)
	at hudson.FilePath$34.invoke(FilePath.java:2557)
	at hudson.FilePath$34.invoke(FilePath.java:2547)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1036)
		at hudson.FilePath.act(FilePath.java:1025)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1819)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1038)
	at hudson.FilePath.act(FilePath.java:1025)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1819)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1583 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1583/

4 tests failed.
FAILED:  org.apache.solr.cloud.LeaderTragicEventTest.test

Error Message:
org.apache.lucene.store.RawDirectoryWrapper cannot be cast to org.apache.lucene.store.MockDirectoryWrapper

Stack Trace:
java.lang.ClassCastException: org.apache.lucene.store.RawDirectoryWrapper cannot be cast to org.apache.lucene.store.MockDirectoryWrapper
	at __randomizedtesting.SeedInfo.seed([16C8F70A7ED0A5B1:9E9CC8D0D02CC849]:0)
	at org.apache.solr.cloud.LeaderTragicEventTest.corruptLeader(LeaderTragicEventTest.java:102)
	at org.apache.solr.cloud.LeaderTragicEventTest.test(LeaderTragicEventTest.java:74)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.LeaderTragicEventTest.testOtherReplicasAreNotActive

Error Message:
Error from server at http://127.0.0.1:45639/solr: collection already exists: collection1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45639/solr: collection already exists: collection1
	at __randomizedtesting.SeedInfo.seed([16C8F70A7ED0A5B1:937CDB7D422F1C29]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.LeaderTragicEventTest.testOtherReplicasAreNotActive(LeaderTragicEventTest.java:139)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.cdcr.CdcrBidirectionalTest.testBiDir

Error Message:
Captured an uncaught exception in thread: Thread[id=12500, name=cdcr-replicator-2883-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=12500, name=cdcr-replicator-2883-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
Caused by: java.lang.AssertionError: 1605795913784098816 != 1605795913704407040
	at __randomizedtesting.SeedInfo.seed([16C8F70A7ED0A5B1]:0)
	at org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
	at org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:125)
	at org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=2 numReplicas=2 but found DocCollection(solrj_collection//collections/solrj_collection/state.json/31)={   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{     "shard1":{       "range":"80000000-ffffffff",       "state":"active",       "replicas":{         "core_node5":{           "dataDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node5/data/",           "base_url":"http://127.0.0.1:39105/g_vfi/j",           "node_name":"127.0.0.1:39105_g_vfi%2Fj",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node5/data/tlog",           "core":"solrj_collection_shard1_replica_n2",           "shared_storage":"true",           "state":"down"},         "core_node3":{           "core":"solrj_collection_shard1_replica_n1",           "base_url":"http://127.0.0.1:40421/g_vfi/j",           "node_name":"127.0.0.1:40421_g_vfi%2Fj",           "state":"down",           "type":"NRT",           "force_set_state":"false"}}},     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node7":{           "dataDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node7/data/",           "base_url":"http://127.0.0.1:36135/g_vfi/j",           "node_name":"127.0.0.1:36135_g_vfi%2Fj",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node7/data/tlog",           "core":"solrj_collection_shard2_replica_n4",           "shared_storage":"true",           "state":"active",           "leader":"true"},         "core_node8":{           "dataDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node8/data/",           "base_url":"http://127.0.0.1:40421/g_vfi/j",           "node_name":"127.0.0.1:40421_g_vfi%2Fj",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node8/data/tlog",           "core":"solrj_collection_shard2_replica_n6",           "shared_storage":"true",           "state":"active"}}}},   "router":{     "field":"myOwnField",     "name":"compositeId"},   "maxShardsPerNode":"2",   "autoAddReplicas":"true",   "nrtReplicas":"2",   "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:36135_g_vfi%2Fj, 127.0.0.1:39105_g_vfi%2Fj, 127.0.0.1:40421_g_vfi%2Fj, 127.0.0.1:45350_g_vfi%2Fj]

Stack Trace:
java.lang.AssertionError: Expected numSlices=2 numReplicas=2 but found DocCollection(solrj_collection//collections/solrj_collection/state.json/31)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node5":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node5/data/",
          "base_url":"http://127.0.0.1:39105/g_vfi/j",
          "node_name":"127.0.0.1:39105_g_vfi%2Fj",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node5/data/tlog",
          "core":"solrj_collection_shard1_replica_n2",
          "shared_storage":"true",
          "state":"down"},
        "core_node3":{
          "core":"solrj_collection_shard1_replica_n1",
          "base_url":"http://127.0.0.1:40421/g_vfi/j",
          "node_name":"127.0.0.1:40421_g_vfi%2Fj",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node7/data/",
          "base_url":"http://127.0.0.1:36135/g_vfi/j",
          "node_name":"127.0.0.1:36135_g_vfi%2Fj",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node7/data/tlog",
          "core":"solrj_collection_shard2_replica_n4",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"},
        "core_node8":{
          "dataDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node8/data/",
          "base_url":"http://127.0.0.1:40421/g_vfi/j",
          "node_name":"127.0.0.1:40421_g_vfi%2Fj",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://lucene2-us-west.apache.org:34824/solr_hdfs_home/solrj_collection/core_node8/data/tlog",
          "core":"solrj_collection_shard2_replica_n6",
          "shared_storage":"true",
          "state":"active"}}}},
  "router":{
    "field":"myOwnField",
    "name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"true",
  "nrtReplicas":"2",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:36135_g_vfi%2Fj, 127.0.0.1:39105_g_vfi%2Fj, 127.0.0.1:40421_g_vfi%2Fj, 127.0.0.1:45350_g_vfi%2Fj]
	at __randomizedtesting.SeedInfo.seed([16C8F70A7ED0A5B1:9E9CC8D0D02CC849]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:415)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:256)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13604 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderTragicEventTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/init-core-data-001
   [junit4]   2> 2118301 WARN  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=130 numCloses=130
   [junit4]   2> 2118301 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2118311 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2118311 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2118312 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001
   [junit4]   2> 2118312 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2118323 INFO  (Thread-5197) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2118323 INFO  (Thread-5197) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2118356 ERROR (Thread-5197) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2118423 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:45365
   [junit4]   2> 2118446 INFO  (zkConnectionManagerCallback-2201-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2118457 INFO  (jetty-launcher-2198-thread-1) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 2118465 INFO  (jetty-launcher-2198-thread-2) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 2118467 INFO  (jetty-launcher-2198-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2118467 INFO  (jetty-launcher-2198-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2118467 INFO  (jetty-launcher-2198-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2118467 INFO  (jetty-launcher-2198-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3fb11fe7{/solr,null,AVAILABLE}
   [junit4]   2> 2118481 INFO  (jetty-launcher-2198-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@491a3fc{HTTP/1.1,[http/1.1]}{127.0.0.1:45691}
   [junit4]   2> 2118481 INFO  (jetty-launcher-2198-thread-2) [    ] o.e.j.s.Server Started @2118538ms
   [junit4]   2> 2118481 INFO  (jetty-launcher-2198-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45691}
   [junit4]   2> 2118481 ERROR (jetty-launcher-2198-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2118481 INFO  (jetty-launcher-2198-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2118481 INFO  (jetty-launcher-2198-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2118481 INFO  (jetty-launcher-2198-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2118481 INFO  (jetty-launcher-2198-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2118481 INFO  (jetty-launcher-2198-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T14:35:48.441Z
   [junit4]   2> 2118483 INFO  (zkConnectionManagerCallback-2203-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2118488 INFO  (jetty-launcher-2198-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2118493 INFO  (jetty-launcher-2198-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2118494 INFO  (jetty-launcher-2198-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2118494 INFO  (jetty-launcher-2198-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2118494 INFO  (jetty-launcher-2198-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5571951{/solr,null,AVAILABLE}
   [junit4]   2> 2118495 INFO  (jetty-launcher-2198-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7809bcc6{HTTP/1.1,[http/1.1]}{127.0.0.1:45639}
   [junit4]   2> 2118495 INFO  (jetty-launcher-2198-thread-1) [    ] o.e.j.s.Server Started @2118552ms
   [junit4]   2> 2118495 INFO  (jetty-launcher-2198-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45639}
   [junit4]   2> 2118495 ERROR (jetty-launcher-2198-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2118495 INFO  (jetty-launcher-2198-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2118495 INFO  (jetty-launcher-2198-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2118495 INFO  (jetty-launcher-2198-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2118495 INFO  (jetty-launcher-2198-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2118495 INFO  (jetty-launcher-2198-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T14:35:48.455Z
   [junit4]   2> 2118524 INFO  (zkConnectionManagerCallback-2205-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2118524 INFO  (jetty-launcher-2198-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2118762 INFO  (jetty-launcher-2198-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45365/solr
   [junit4]   2> 2118783 INFO  (zkConnectionManagerCallback-2209-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2118819 INFO  (zkConnectionManagerCallback-2211-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2118995 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2118996 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45639_solr
   [junit4]   2> 2118999 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.c.Overseer Overseer (id=72958291280658436-127.0.0.1:45639_solr-n_0000000000) starting
   [junit4]   2> 2119019 INFO  (zkConnectionManagerCallback-2218-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2119021 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45365/solr ready
   [junit4]   2> 2119022 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45639_solr
   [junit4]   2> 2119061 INFO  (zkCallback-2217-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2119072 INFO  (OverseerStateUpdate-72958291280658436-127.0.0.1:45639_solr-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2119095 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2119134 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45639.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2119153 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45639.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2119153 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45639.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2119154 INFO  (jetty-launcher-2198-thread-1) [n:127.0.0.1:45639_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001/node1/.
   [junit4]   2> 2119287 INFO  (jetty-launcher-2198-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45365/solr
   [junit4]   2> 2119308 INFO  (zkConnectionManagerCallback-2223-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2119331 INFO  (zkConnectionManagerCallback-2225-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2119337 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2119338 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2119340 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2119340 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45691_solr
   [junit4]   2> 2119343 INFO  (zkCallback-2210-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2119343 INFO  (zkCallback-2217-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2119353 INFO  (zkCallback-2224-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2119467 INFO  (zkConnectionManagerCallback-2232-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2119470 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2119476 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45365/solr ready
   [junit4]   2> 2119499 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2119530 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45691.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2119558 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45691.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2119558 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45691.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2119559 INFO  (jetty-launcher-2198-thread-2) [n:127.0.0.1:45691_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001/node2/.
   [junit4]   2> 2119679 INFO  (zkConnectionManagerCallback-2235-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2119682 INFO  (zkConnectionManagerCallback-2240-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2119683 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2119684 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45365/solr ready
   [junit4]   2> 2119771 INFO  (TEST-LeaderTragicEventTest.test-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 2119774 INFO  (qtp474573419-12056) [n:127.0.0.1:45639_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=config&name=collection1&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2119791 INFO  (OverseerThreadFactory-2754-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2119912 INFO  (OverseerStateUpdate-72958291280658436-127.0.0.1:45639_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:45639/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2119914 INFO  (OverseerStateUpdate-72958291280658436-127.0.0.1:45639_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:45691/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2120131 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr    x:collection1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2120132 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr    x:collection1_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2120143 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr    x:collection1_shard1_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n2&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2121205 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2121205 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2121230 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.s.IndexSchema [collection1_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 2121232 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2121232 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n2' using configuration from collection collection1, trusted=true
   [junit4]   2> 2121233 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45691.solr.core.collection1.shard1.replica_n2' (registry 'solr.core.collection1.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2121233 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2121233 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SolrCore [[collection1_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001/node2/collection1_shard1_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001/node2/./collection1_shard1_replica_n2/data/]
   [junit4]   2> 2121237 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2121239 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2121239 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 2121280 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45639.solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2121289 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2121289 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001/node1/collection1_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001/node1/./collection1_shard1_replica_n1/data/]
   [junit4]   2> 2121447 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2121447 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2121448 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2121448 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2121458 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@71f28c4a[collection1_shard1_replica_n2] main]
   [junit4]   2> 2121459 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2121459 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2121461 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2121461 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2121462 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 2121463 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 2121463 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2121464 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@705ed6d3[collection1_shard1_replica_n1] main]
   [junit4]   2> 2121464 INFO  (searcherExecutor-2765-thread-1-processing-n:127.0.0.1:45691_solr x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SolrCore [collection1_shard1_replica_n2] Registered new searcher Searcher@71f28c4a[collection1_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2121464 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1605795736635572224
   [junit4]   2> 2121465 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 2121465 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 2121466 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2121466 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1605795736637669376
   [junit4]   2> 2121467 INFO  (searcherExecutor-2766-thread-1-processing-n:127.0.0.1:45639_solr x:collection1_shard1_replica_n1 c:collection1 s:shard1 r:core_node3) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@705ed6d3[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2121473 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node4=0}, version=0}
   [junit4]   2> 2121476 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node3=0, core_node4=0}, version=1}
   [junit4]   2> 2121479 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2121479 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2121479 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45691/solr/collection1_shard1_replica_n2/
   [junit4]   2> 2121480 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n2 url=http://127.0.0.1:45691/solr START replicas=[http://127.0.0.1:45639/solr/collection1_shard1_replica_n1/] nUpdates=100
   [junit4]   2> 2121499 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n2 url=http://127.0.0.1:45691/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 2121501 INFO  (qtp474573419-12051) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 2121502 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2121502 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2121502 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2121507 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45691/solr/collection1_shard1_replica_n2/ shard1
   [junit4]   2> 2121614 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2121616 INFO  (qtp98004118-12044) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n2&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1473
   [junit4]   2> 2121717 INFO  (zkCallback-2224-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2122483 INFO  (qtp474573419-12052) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2352
   [junit4]   2> 2122499 INFO  (qtp474573419-12056) [n:127.0.0.1:45639_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2122592 INFO  (zkCallback-2210-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2122593 INFO  (zkCallback-2224-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2123499 INFO  (qtp474573419-12056) [n:127.0.0.1:45639_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=config&name=collection1&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3725
   [junit4]   2> 2123502 INFO  (TEST-LeaderTragicEventTest.test-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending test
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderTragicEventTest -Dtests.method=test -Dtests.seed=16C8F70A7ED0A5B1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=es-EC -Dtests.timezone=America/Kentucky/Louisville -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   3.82s J2 | LeaderTragicEventTest.test <<<
   [junit4]    > Throwable #1: java.lang.ClassCastException: org.apache.lucene.store.RawDirectoryWrapper cannot be cast to org.apache.lucene.store.MockDirectoryWrapper
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([16C8F70A7ED0A5B1:9E9CC8D0D02CC849]:0)
   [junit4]    > 	at org.apache.solr.cloud.LeaderTragicEventTest.corruptLeader(LeaderTragicEventTest.java:102)
   [junit4]    > 	at org.apache.solr.cloud.LeaderTragicEventTest.test(LeaderTragicEventTest.java:74)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2123660 INFO  (TEST-LeaderTragicEventTest.testOtherReplicasAreNotActive-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testOtherReplicasAreNotActive
   [junit4]   2> 2123660 INFO  (qtp474573419-12051) [n:127.0.0.1:45639_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=config&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2123681 INFO  (OverseerCollectionConfigSetProcessor-72958291280658436-127.0.0.1:45639_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2123681 INFO  (OverseerThreadFactory-2754-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 2123682 ERROR (OverseerThreadFactory-2754-thread-2) [    ] o.a.s.c.a.c.OverseerCollectionMessageHandler Collection: collection1 operation: create failed:org.apache.solr.common.SolrException: collection already exists: collection1
   [junit4]   2> 	at org.apache.solr.cloud.api.collections.CreateCollectionCmd.call(CreateCollectionCmd.java:106)
   [junit4]   2> 	at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:255)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:469)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 2123702 INFO  (qtp474573419-12051) [n:127.0.0.1:45639_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=config&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=400 QTime=41
   [junit4]   2> 2123703 INFO  (TEST-LeaderTragicEventTest.testOtherReplicasAreNotActive-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testOtherReplicasAreNotActive
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderTragicEventTest -Dtests.method=testOtherReplicasAreNotActive -Dtests.seed=16C8F70A7ED0A5B1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=es-EC -Dtests.timezone=America/Kentucky/Louisville -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.20s J2 | LeaderTragicEventTest.testOtherReplicasAreNotActive <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45639/solr: collection already exists: collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([16C8F70A7ED0A5B1:937CDB7D422F1C29]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.cloud.LeaderTragicEventTest.testOtherReplicasAreNotActive(LeaderTragicEventTest.java:139)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2123731 INFO  (jetty-closer-2199-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@7809bcc6{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2123735 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2145643617
   [junit4]   2> 2123735 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2123736 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@54f049ef: rootName = solr_45639, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@5a924391
   [junit4]   2> 2123736 INFO  (jetty-closer-2199-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@491a3fc{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2123750 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=745251544
   [junit4]   2> 2123750 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2123750 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@771dedfc: rootName = solr_45691, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@7c52438f
   [junit4]   2> 2123756 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2123756 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2cf3fe7: rootName = solr_45691, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@7080b682
   [junit4]   2> 2123758 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2123758 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4418fc47: rootName = solr_45639, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@7080b682
   [junit4]   2> 2123759 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2123759 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2bd1681c: rootName = solr_45691, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@2a5b1277
   [junit4]   2> 2123762 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:45691_solr
   [junit4]   2> 2123763 INFO  (zkCallback-2217-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2123763 INFO  (zkCallback-2210-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2123764 INFO  (jetty-closer-2199-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2123778 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2123778 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7ff7e23a: rootName = solr_45639, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@2a5b1277
   [junit4]   2> 2123778 INFO  (zkCallback-2231-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2123778 INFO  (coreCloseExecutor-2775-thread-1) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SolrCore [collection1_shard1_replica_n2]  CLOSING SolrCore org.apache.solr.core.SolrCore@7c1850f5
   [junit4]   2> 2123778 INFO  (zkCallback-2224-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2123778 INFO  (coreCloseExecutor-2775-thread-1) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.replica_n2, tag=7c1850f5
   [junit4]   2> 2123779 INFO  (coreCloseExecutor-2775-thread-1) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2216ccf5: rootName = solr_45691, domain = solr.core.collection1.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.collection1.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@3bdcfbe1
   [junit4]   2> 2123779 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:45639_solr
   [junit4]   2> 2123780 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2123788 INFO  (coreCloseExecutor-2775-thread-1) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=7c1850f5
   [junit4]   2> 2123811 INFO  (coreCloseExecutor-2776-thread-1) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3a4ddf1a
   [junit4]   2> 2123812 INFO  (coreCloseExecutor-2776-thread-1) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.replica_n1, tag=3a4ddf1a
   [junit4]   2> 2123812 INFO  (coreCloseExecutor-2776-thread-1) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@59dbb344: rootName = solr_45639, domain = solr.core.collection1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.collection1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@37ad4e1c
   [junit4]   2> 2123812 INFO  (zkCallback-2210-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2123812 INFO  (zkCallback-2217-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2123839 INFO  (zkCallback-2231-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2123931 INFO  (zkCallback-2224-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2123933 INFO  (coreCloseExecutor-2776-thread-1) [n:127.0.0.1:45639_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=3a4ddf1a
   [junit4]   2> 2123934 INFO  (zkCallback-2210-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 2123949 INFO  (zkCallback-2224-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 2123988 INFO  (jetty-closer-2199-thread-1) [    ] o.a.s.c.Overseer Overseer (id=72958291280658436-127.0.0.1:45639_solr-n_0000000000) closing
   [junit4]   2> 2123993 INFO  (OverseerStateUpdate-72958291280658436-127.0.0.1:45639_solr-n_0000000000) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45639_solr
   [junit4]   2> 2123998 WARN  (OverseerAutoScalingTriggerThread-72958291280658436-127.0.0.1:45639_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 2124005 INFO  (zkCallback-2224-thread-2) [    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45691_solr
   [junit4]   2> 2124006 INFO  (jetty-closer-2199-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5571951{/solr,null,UNAVAILABLE}
   [junit4]   2> 2124007 INFO  (jetty-closer-2199-thread-1) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 2136559 ERROR (coreCloseExecutor-2775-thread-1) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.CachingDirectoryFactory Timeout waiting for all directory ref counts to be released - gave up waiting on CachedDir<<refCount=1;path=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001/node2/./collection1_shard1_replica_n2/data/index;done=false>>
   [junit4]   2> 2136560 ERROR (coreCloseExecutor-2775-thread-1) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.CachingDirectoryFactory Error closing directory:org.apache.solr.common.SolrException: Timeout waiting for all directory ref counts to be released - gave up waiting on CachedDir<<refCount=1;path=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001/tempDir-001/node2/./collection1_shard1_replica_n2/data/index;done=false>>
   [junit4]   2> 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:178)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1643)
   [junit4]   2> 	at org.apache.solr.core.SolrCores.lambda$close$0(SolrCores.java:141)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 2136560 ERROR (coreCloseExecutor-2775-thread-1) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SolrCore java.lang.AssertionError: 1
   [junit4]   2> 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:192)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1643)
   [junit4]   2> 	at org.apache.solr.core.SolrCores.lambda$close$0(SolrCores.java:141)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 2136560 ERROR (coreCloseExecutor-2775-thread-1) [n:127.0.0.1:45691_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] o.a.s.c.SolrCores Error shutting down core:java.lang.AssertionError: 1
   [junit4]   2> 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:192)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1643)
   [junit4]   2> 	at org.apache.solr.core.SolrCores.lambda$close$0(SolrCores.java:141)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 2136591 INFO  (jetty-closer-2199-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3fb11fe7{/solr,null,UNAVAILABLE}
   [junit4]   2> 2136591 INFO  (jetty-closer-2199-thread-2) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 2136612 ERROR (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2136613 INFO  (SUITE-LeaderTragicEventTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45365 45365
   [junit4]   2> 2136669 INFO  (Thread-5197) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45365 45365
   [junit4]   2> 2136685 WARN  (Thread-5197) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/collections/collection1/terms/shard1
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/configs/config
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/live_nodes
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_16C8F70A7ED0A5B1-001
   [junit4]   2> Jul 12, 2018 2:36:06 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1262, maxMBSortInHeap=6.474910515687563, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@71d0d66f), locale=es-EC, timezone=America/Kentucky/Louisville
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=288957864,total=508559360
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsChaosMonkeyNothingIsSafeTest, TestReloadAndDeleteDocs, RAMDirectoryFactoryTest, TestSurroundQueryParser, TestIntervalFaceting, QueryElevationComponentTest, TestSortByMinMaxFunction, FieldMutatingUpdateProcessorTest, TestTestInjection, TestRecovery, TestNumericTokenStream, PropertiesRequestHandlerTest, ExplicitHLLTest, SolrTestCaseJ4Test, TestSlowCompositeReaderWrapper, TestSolrConfigHandlerConcurrent, TestScoreJoinQPNoScore, CloudMLTQParserTest, TestApiFramework, StatsComponentTest, TermVectorComponentDistributedTest, NumberUtilsTest, GraphQueryTest, TestComponentsName, TestFieldCollectionResource, MultiSolrCloudTestCaseTest, DistributedQueueTest, TestSimpleQParserPlugin, TestImpersonationWithHadoopAuth, TestNRTOpen, CoreAdminOperationTest, BlockJoinFacetDistribTest, TestNestedUpdateProcessor, TestConfig, TestCloudSearcherWarming, ScriptEngineTest, TestLRUCache, TestLockTree, TestPartialUpdateDeduplication, TestSolrConfigHandler, TestDistributedStatsComponentCardinality, TestImplicitCoreProperties, TestMaxTokenLenTokenizer, DeleteReplicaTest, HdfsAutoAddReplicasIntegrationTest, TestStressCloudBlindAtomicUpdates, CdcrReplicationHandlerTest, LeaderTragicEventTest]
   [junit4] Completed [150/824 (1!)] on J2 in 18.87s, 2 tests, 2 errors <<< FAILURES!

[...truncated 70 lines...]
   [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBidirectionalTest
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_16C8F70A7ED0A5B1-001/init-core-data-001
   [junit4]   2> 2174272 WARN  (SUITE-CdcrBidirectionalTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 2174272 INFO  (SUITE-CdcrBidirectionalTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2174273 INFO  (SUITE-CdcrBidirectionalTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2174273 INFO  (SUITE-CdcrBidirectionalTest-seed#[16C8F70A7ED0A5B1]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2174309 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBiDir
   [junit4]   2> 2174309 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_16C8F70A7ED0A5B1-001/cdcr-cluster2-001
   [junit4]   2> 2174309 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2174339 INFO  (Thread-5245) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2174339 INFO  (Thread-5245) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2174356 ERROR (Thread-5245) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2174436 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.c.ZkTestServer start zk server on port:41858
   [junit4]   2> 2174526 INFO  (zkConnectionManagerCallback-2289-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2174600 INFO  (jetty-launcher-2286-thread-1) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 2174686 INFO  (jetty-launcher-2286-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2174686 INFO  (jetty-launcher-2286-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2174686 INFO  (jetty-launcher-2286-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2174705 INFO  (jetty-launcher-2286-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4a9732db{/solr,null,AVAILABLE}
   [junit4]   2> 2174706 INFO  (jetty-launcher-2286-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@21d34470{HTTP/1.1,[http/1.1]}{127.0.0.1:41206}
   [junit4]   2> 2174706 INFO  (jetty-launcher-2286-thread-1) [    ] o.e.j.s.Server Started @2174763ms
   [junit4]   2> 2174706 INFO  (jetty-launcher-2286-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41206}
   [junit4]   2> 2174706 ERROR (jetty-launcher-2286-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2174706 INFO  (jetty-launcher-2286-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2174706 INFO  (jetty-launcher-2286-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2174706 INFO  (jetty-launcher-2286-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2174706 INFO  (jetty-launcher-2286-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2174706 INFO  (jetty-launcher-2286-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T14:36:44.666Z
   [junit4]   2> 2174782 INFO  (zkConnectionManagerCallback-2291-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2174794 INFO  (jetty-launcher-2286-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2176089 INFO  (jetty-launcher-2286-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41858/solr
   [junit4]   2> 2176158 INFO  (zkConnectionManagerCallback-2295-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2176195 INFO  (zkConnectionManagerCallback-2297-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2176692 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2176692 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41206_solr
   [junit4]   2> 2176693 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.c.Overseer Overseer (id=72958294950805507-127.0.0.1:41206_solr-n_0000000000) starting
   [junit4]   2> 2176772 INFO  (zkConnectionManagerCallback-2304-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2176792 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41858/solr ready
   [junit4]   2> 2176856 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41206_solr
   [junit4]   2> 2177104 INFO  (zkCallback-2303-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2177123 INFO  (zkCallback-2296-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2177250 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2177741 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41206.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2177886 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41206.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2177887 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41206.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2177888 INFO  (jetty-launcher-2286-thread-1) [n:127.0.0.1:41206_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_16C8F70A7ED0A5B1-001/cdcr-cluster2-001/node1/.
   [junit4]   2> 2178259 INFO  (zkConnectionManagerCallback-2307-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2178345 INFO  (zkConnectionManagerCallback-2310-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2178418 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_16C8F70A7ED0A5B1-001/cdcr-cluster1-001
   [junit4]   2> 2178418 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2178419 INFO  (Thread-5255) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2178419 INFO  (Thread-5255) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2178439 ERROR (Thread-5255) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2178527 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.c.ZkTestServer start zk server on port:40925
   [junit4]   2> 2178584 INFO  (zkConnectionManagerCallback-2314-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2178603 INFO  (jetty-launcher-2311-thread-1) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 2178657 INFO  (jetty-launcher-2311-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2178657 INFO  (jetty-launcher-2311-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2178657 INFO  (jetty-launcher-2311-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 2178658 INFO  (jetty-launcher-2311-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@79bf1966{/solr,null,AVAILABLE}
   [junit4]   2> 2178658 INFO  (jetty-launcher-2311-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2de13795{HTTP/1.1,[http/1.1]}{127.0.0.1:46615}
   [junit4]   2> 2178658 INFO  (jetty-launcher-2311-thread-1) [    ] o.e.j.s.Server Started @2178715ms
   [junit4]   2> 2178658 INFO  (jetty-launcher-2311-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46615}
   [junit4]   2> 2178658 ERROR (jetty-launcher-2311-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2178666 INFO  (jetty-launcher-2311-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2178666 INFO  (jetty-launcher-2311-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2178666 INFO  (jetty-launcher-2311-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2178667 INFO  (jetty-launcher-2311-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2178667 INFO  (jetty-launcher-2311-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T14:36:48.627Z
   [junit4]   2> 2178722 INFO  (zkConnectionManagerCallback-2316-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2178722 INFO  (jetty-launcher-2311-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2179340 INFO  (jetty-launcher-2311-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40925/solr
   [junit4]   2> 2179412 INFO  (zkConnectionManagerCallback-2320-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2179477 INFO  (zkConnectionManagerCallback-2322-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2180335 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2180336 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46615_solr
   [junit4]   2> 2180336 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.c.Overseer Overseer (id=72958295219306499-127.0.0.1:46615_solr-n_0000000000) starting
   [junit4]   2> 2180471 INFO  (zkConnectionManagerCallback-2329-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2180617 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40925/solr ready
   [junit4]   2> 2180654 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46615_solr
   [junit4]   2> 2180691 INFO  (OverseerStateUpdate-72958295219306499-127.0.0.1:46615_solr-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2180922 INFO  (zkCallback-2328-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2181227 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2181451 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46615.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2181544 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46615.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2181544 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46615.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3381fa56
   [junit4]   2> 2181545 INFO  (jetty-launcher-2311-thread-1) [n:127.0.0.1:46615_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_16C8F70A7ED0A5B1-001/cdcr-cluster1-001/node1/.
   [junit4]   2> 2181881 INFO  (zkConnectionManagerCallback-2332-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2181918 INFO  (zkConnectionManagerCallback-2335-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2181919 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.c.c.CdcrBidirectionalTest cluster2 zkHost = 127.0.0.1:41858/solr
   [junit4]   2> 2181920 INFO  (TEST-CdcrBidirectionalTest.testBiDir-seed#[16C8F70A7ED0A5B1]) [    ] o.a.s.c.c.CdcrBidirectionalTest cluster1 zkHost = 127.0.0.1:40925/solr
   [junit4]   2> 2181991 INFO  (zkConnectionManagerCallback-2337-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2182044 INFO  (zkConnectionManagerCallback-2341-thread-1) [    ] 

[...truncated too long message...]

source] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null2060716911
     [copy] Copying 241 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null2060716911
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null2060716911

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 30ms :: artifacts dl 5ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 529 minutes 16 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$34.hasMatch(FilePath.java:2678)
	at hudson.FilePath$34.invoke(FilePath.java:2557)
	at hudson.FilePath$34.invoke(FilePath.java:2547)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1036)
		at hudson.FilePath.act(FilePath.java:1025)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1819)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1038)
	at hudson.FilePath.act(FilePath.java:1025)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1819)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)