You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/03/26 11:24:36 UTC

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 183 - Failure

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/183/

3 tests failed.
FAILED:  org.apache.lucene.index.TestIndexSorting.testRandom3

Error Message:
this IndexWriter is closed

Stack Trace:
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
	at __randomizedtesting.SeedInfo.seed([201B75CF428D0D9B:82C33B15267F249D]:0)
	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:899)
	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:913)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1769)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1463)
	at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2299)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Java heap space
	at org.apache.lucene.util.packed.Packed8ThreeBlocks.<init>(Packed8ThreeBlocks.java:41)
	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:963)
	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
	at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
	at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
	at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
	at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
	at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
	at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
	at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
	at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
	at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
	at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
	at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4477)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)


FAILED:  org.apache.lucene.index.TestIndexSorting.testMultiValuedRandom1

Error Message:
Captured an uncaught exception in thread: Thread[id=22391, name=Lucene Merge Thread #16108, state=RUNNABLE, group=TGRP-TestIndexSorting]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=22391, name=Lucene Merge Thread #16108, state=RUNNABLE, group=TGRP-TestIndexSorting]
	at __randomizedtesting.SeedInfo.seed([201B75CF428D0D9B:EEF6C01D972A1123]:0)
Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([201B75CF428D0D9B]:0)
	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
Caused by: java.lang.OutOfMemoryError: Java heap space
	at org.apache.lucene.util.packed.Packed8ThreeBlocks.<init>(Packed8ThreeBlocks.java:41)
	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:963)
	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
	at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
	at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
	at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
	at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
	at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
	at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
	at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
	at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
	at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
	at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
	at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4477)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:42755: ADDREPLICA failed to create replica

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:42755: ADDREPLICA failed to create replica
	at __randomizedtesting.SeedInfo.seed([46ACA90D8DC97E65:CEF896D72335139D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:120)
	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:72)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 1887 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J0-20180326_064007_5474704563399186278877.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] codec: CheapBastard, pf: FSTOrd50, dvf: Direct
   [junit4] <<< JVM J0: EOF ----

[...truncated 7 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexSorting
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=201B75CF428D0D9B -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=be -Dtests.timezone=America/Bahia_Banderas -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   1796s J2 | TestIndexSorting.testRandom3 <<<
   [junit4]    > Throwable #1: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([201B75CF428D0D9B:82C33B15267F249D]:0)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:899)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:913)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1769)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1463)
   [junit4]    > 	at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2299)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at org.apache.lucene.util.packed.Packed8ThreeBlocks.<init>(Packed8ThreeBlocks.java:41)
   [junit4]    > 	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:963)
   [junit4]    > 	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
   [junit4]    > 	at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
   [junit4]    > 	at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
   [junit4]    > 	at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
   [junit4]    > 	at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
   [junit4]    > 	at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
   [junit4]    > 	at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
   [junit4]    > 	at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
   [junit4]    > 	at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4477)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> ??? 26, 2018 2:04:02 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #16108,5,TGRP-TestIndexSorting]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([201B75CF428D0D9B]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2> 	at org.apache.lucene.util.packed.Packed8ThreeBlocks.<init>(Packed8ThreeBlocks.java:41)
   [junit4]   2> 	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:963)
   [junit4]   2> 	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
   [junit4]   2> 	at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
   [junit4]   2> 	at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
   [junit4]   2> 	at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
   [junit4]   2> 	at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
   [junit4]   2> 	at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
   [junit4]   2> 	at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
   [junit4]   2> 	at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
   [junit4]   2> 	at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
   [junit4]   2> 	at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
   [junit4]   2> 	at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
   [junit4]   2> 	at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4477)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexSorting -Dtests.method=testMultiValuedRandom1 -Dtests.seed=201B75CF428D0D9B -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=be -Dtests.timezone=America/Bahia_Banderas -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7.76s J2 | TestIndexSorting.testMultiValuedRandom1 <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=22391, name=Lucene Merge Thread #16108, state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([201B75CF428D0D9B:EEF6C01D972A1123]:0)
   [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([201B75CF428D0D9B]:0)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at org.apache.lucene.util.packed.Packed8ThreeBlocks.<init>(Packed8ThreeBlocks.java:41)
   [junit4]    > 	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:963)
   [junit4]    > 	at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
   [junit4]    > 	at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
   [junit4]    > 	at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
   [junit4]    > 	at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
   [junit4]    > 	at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
   [junit4]    > 	at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
   [junit4]    > 	at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:296)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.add(Builder.java:400)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:373)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:416)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:406)
   [junit4]    > 	at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:527)
   [junit4]    > 	at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4477)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexSorting_201B75CF428D0D9B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {docs=Lucene50(blocksize=128), sparse_text=Lucene50(blocksize=128), norms=Lucene50(blocksize=128), positions=Lucene50(blocksize=128), id=Lucene50(blocksize=128), term_vectors=PostingsFormat(name=Memory)}, docValues:{multi_valued_long=DocValuesFormat(name=Lucene70), foo=DocValuesFormat(name=Lucene70), numeric=DocValuesFormat(name=Asserting), float=DocValuesFormat(name=Lucene70), multi_valued_numeric=DocValuesFormat(name=Memory), long=DocValuesFormat(name=Lucene70), points=DocValuesFormat(name=Memory), sparse_int=DocValuesFormat(name=Lucene70), bar=DocValuesFormat(name=Memory), multi_valued_double=DocValuesFormat(name=Memory), docs=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Memory), multi_valued_int=DocValuesFormat(name=Asserting), multi_valued_bytes=DocValuesFormat(name=Asserting), sparse_text=DocValuesFormat(name=Lucene70), double=DocValuesFormat(name=Asserting), positions=DocValuesFormat(name=Lucene70), sparse_binary=DocValuesFormat(name=Lucene70), int=DocValuesFormat(name=Memory), sorted=DocValuesFormat(name=Asserting), sparse=DocValuesFormat(name=Lucene70), multi_valued_string=DocValuesFormat(name=Memory), norms=DocValuesFormat(name=Memory), bytes=DocValuesFormat(name=Memory), binary=DocValuesFormat(name=Asserting), dense_int=DocValuesFormat(name=Memory), multi_valued_float=DocValuesFormat(name=Lucene70), term_vectors=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=71, maxMBSortInHeap=6.149716729696173, sim=RandomSimilarity(queryNorm=true): {positions=IB SPL-D1, id=IB LL-D2, term_vectors=DFR I(n)2}, locale=be, timezone=America/Bahia_Banderas
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=139988896,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [TestPositiveScoresOnlyCollector, TestLSBRadixSorter, TestSloppyPhraseQuery2, TestSentinelIntSet, TestIndexReaderClose, TestDocValuesQueries, TestIndexWriterCommit, TestBasics, TestCharsRef, TestFutureArrays, Test2BPoints, TestReaderClosed, TestSimilarityBase, Test2BBKDPoints, TestAllFilesCheckIndexHeader, TestAutomaton, TestShardSearching, TestBytesRefHash, TestMatchNoDocsQuery, TestDocument, TestNRTReaderWithThreads, TestLucene50StoredFieldsFormatHighCompression, TestSloppyMath, TestSegmentReader, TestScoreCachingWrappingScorer, TestFieldValueQuery, TestStressAdvance, TestHighCompressionMode, TestHugeRamFile, TestReadOnlyIndex, TestFastCompressionMode, TestDocIDMerger, TestBufferedChecksum, TestFieldsReader, TestPayloads, TestSimpleExplanationsOfNonMatches, TestLongPostings, TestTimSorterWorstCase, TestCachingTokenFilter, TestTragicIndexWriterDeadlock, TestTopDocsMerge, TestForUtil, TestParallelCompositeReader, TestDateSort, TestDoubleValuesSource, TestLazyProxSkipping, TestIndexWriterThreadsToSegments, TestFilterCodecReader, TestGeoUtils, TestMultiTermsEnum, TestBytesRefArray, TestCompiledAutomaton, TestMutablePointsReaderUtils, TestNormsFieldExistsQuery, TestSpanContainQuery, TestBoostQuery, TestDocsWithFieldSet, TestBinaryDocument, TestSearchWithThreads, TestRollingBuffer, TestCharTermAttributeImpl, TestIndexWriterLockRelease, TestIndexWriterReader, Test2BNumericDocValues, TestIndexCommit, TestIndexWriterMergePolicy, TestTermContext, TestBlockPostingsFormat, TestConsistentFieldNumbers, TestNewestSegment, TestSortedSetDocValues, FuzzyTermOnShortTermsTest, TestPayloadsOnVectors, TestByteBlockPool, TestFixedBitDocIdSet, TestRegexpQuery, TestSpanSearchEquivalence, TestTermdocPerf, TestIndexWriter, TestIndexWriterMerging, TestFSTs, TestBytesStore, TestPackedInts, TestIndexWriterWithThreads, TestGraphTokenizers, TestAddIndexes, TestIndexWriterExceptions, TestMultiMMap, TestSearcherManager, TestDuelingCodecs, TestBoolean2, TestPersistentSnapshotDeletionPolicy, TestAtomicUpdate, TestNorms, TestDirectoryReaderReopen, TestTimeLimitingCollector, TestConcurrentMergeScheduler, TestStressIndexing, TestRollingUpdates, TestSnapshotDeletionPolicy, TestIndexWriterUnicode, TestIndexWriterForceMerge, TestPerFieldPostingsFormat, TestArrayUtil, TestBufferedIndexInput, TestSimpleExplanations, TestDirectoryReader, TestLockFactory, TestSegmentMerger, TestCollectionUtil, TestBagOfPositions, TestLucene70NormsFormat, TestExitableDirectoryReader, TestFieldReuse, TestFilterDirectoryReader, TestIndexSorting]
   [junit4] Completed [467/467 (1!)] on J2 in 1832.32s, 50 tests, 2 errors <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J2-20180326_064007_5479191407727774129698.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid11603.hprof ...
   [junit4] Heap dump file created [417013572 bytes in 1.471 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 12866 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/init-core-data-001
   [junit4]   2> 4542556 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 4542556 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 4542559 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 4542559 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 4542559 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 4542624 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 4542644 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4542648 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4542678 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_33583_hdfs____4mx06y/webapp
   [junit4]   2> 4543144 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33583
   [junit4]   2> 4543257 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4543258 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4543277 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37111_datanode____eeksr8/webapp
   [junit4]   2> 4543756 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37111
   [junit4]   2> 4543833 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4543851 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 4543871 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_42269_datanode____nb71r2/webapp
   [junit4]   2> 4543923 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:41318) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 4543948 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x4f46a7f3e8d6cb: from storage DS-99e5ebbf-2965-49cc-8b04-4bc40088b5f1 node DatanodeRegistration(127.0.0.1:34921, datanodeUuid=ec847861-ef8d-4ddd-9716-fd0956c64d00, infoPort=56874, infoSecurePort=0, ipcPort=57883, storageInfo=lv=-56;cid=testClusterID;nsid=616030845;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 4543948 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x4f46a7f3e8d6cb: from storage DS-069a2087-b8c5-4c49-8515-76936b23b188 node DatanodeRegistration(127.0.0.1:34921, datanodeUuid=ec847861-ef8d-4ddd-9716-fd0956c64d00, infoPort=56874, infoSecurePort=0, ipcPort=57883, storageInfo=lv=-56;cid=testClusterID;nsid=616030845;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 4544380 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42269
   [junit4]   2> 4544555 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:41318) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 4544575 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x4f46a81973ab3a: from storage DS-c747181d-0b6c-42c9-a8c7-cb5f3b555b7e node DatanodeRegistration(127.0.0.1:33038, datanodeUuid=9568d52c-56cc-454e-ae12-28b007afbf44, infoPort=44554, infoSecurePort=0, ipcPort=45563, storageInfo=lv=-56;cid=testClusterID;nsid=616030845;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 4544575 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x4f46a81973ab3a: from storage DS-707dc755-ade2-429f-b5a1-117531794ff4 node DatanodeRegistration(127.0.0.1:33038, datanodeUuid=9568d52c-56cc-454e-ae12-28b007afbf44, infoPort=44554, infoSecurePort=0, ipcPort=45563, storageInfo=lv=-56;cid=testClusterID;nsid=616030845;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 4544965 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4544965 INFO  (Thread-8791) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4544965 INFO  (Thread-8791) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4544973 ERROR (Thread-8791) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4545065 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.ZkTestServer start zk server on port:42973
   [junit4]   2> 4545074 INFO  (zkConnectionManagerCallback-2664-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4545091 INFO  (zkConnectionManagerCallback-2666-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4545105 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4545108 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 4545112 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4545114 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4545116 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4545118 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4545120 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4545124 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4545126 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4545128 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4545130 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4545134 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 4545299 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T08:27:37+11:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 4545300 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4545300 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4545300 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 4545301 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@489e5d63{/,null,AVAILABLE}
   [junit4]   2> 4545302 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7c5d802d{HTTP/1.1,[http/1.1]}{127.0.0.1:56693}
   [junit4]   2> 4545302 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.Server Started @4545354ms
   [junit4]   2> 4545302 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:41318/hdfs__localhost_41318__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=56693, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/control-001/cores}
   [junit4]   2> 4545302 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4545302 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4545302 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.4.0
   [junit4]   2> 4545302 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4545302 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4545302 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-26T11:05:09.086Z
   [junit4]   2> 4545304 INFO  (zkConnectionManagerCallback-2668-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4545306 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4545306 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/control-001/solr.xml
   [junit4]   2> 4545310 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4545311 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4545312 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4545317 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42973/solr
   [junit4]   2> 4545319 INFO  (zkConnectionManagerCallback-2672-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4545322 INFO  (zkConnectionManagerCallback-2674-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4545441 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4545442 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56693_
   [junit4]   2> 4545443 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.c.Overseer Overseer (id=73519978192896004-127.0.0.1:56693_-n_0000000000) starting
   [junit4]   2> 4545454 INFO  (zkConnectionManagerCallback-2679-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4545457 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42973/solr ready
   [junit4]   2> 4545458 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56693_
   [junit4]   2> 4545466 INFO  (OverseerStateUpdate-73519978192896004-127.0.0.1:56693_-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4545483 INFO  (zkCallback-2678-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4546888 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465
   [junit4]   2> 4546904 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465
   [junit4]   2> 4546904 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465
   [junit4]   2> 4546907 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:56693_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/control-001/cores
   [junit4]   2> 4546984 INFO  (zkConnectionManagerCallback-2683-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4546989 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4546991 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42973/solr ready
   [junit4]   2> 4546998 INFO  (qtp471384468-19412) [n:127.0.0.1:56693_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56693_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4547004 INFO  (OverseerThreadFactory-5648-thread-1) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 4547126 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4547127 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4547237 INFO  (zkCallback-2673-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4548160 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 4548178 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 4548315 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 4548333 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 4548334 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465
   [junit4]   2> 4548334 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:41318/solr_hdfs_home
   [junit4]   2> 4548334 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 4548334 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4548334 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:41318/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 4548336 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:41318/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 4548345 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4548345 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 4548345 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 4548365 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4548367 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:41318/solr_hdfs_home/control_collection/core_node2/data
   [junit4]   2> 4548392 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:41318/solr_hdfs_home/control_collection/core_node2/data/index
   [junit4]   2> 4548401 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4548401 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 4548401 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 4548417 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4548418 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.21955043209206915]
   [junit4]   2> 4548452 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34921 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-707dc755-ade2-429f-b5a1-117531794ff4:NORMAL:127.0.0.1:33038|RBW], ReplicaUC[[DISK]DS-99e5ebbf-2965-49cc-8b04-4bc40088b5f1:NORMAL:127.0.0.1:34921|RBW]]} size 0
   [junit4]   2> 4548457 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33038 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-99e5ebbf-2965-49cc-8b04-4bc40088b5f1:NORMAL:127.0.0.1:34921|RBW], ReplicaUC[[DISK]DS-c747181d-0b6c-42c9-a8c7-cb5f3b555b7e:NORMAL:127.0.0.1:33038|FINALIZED]]} size 0
   [junit4]   2> 4548469 WARN  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 4548535 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 4548535 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4548535 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 4548547 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4548547 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4548566 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4480062527442674]
   [junit4]   2> 4548573 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6345b93c[control_collection_shard1_replica_n1] main]
   [junit4]   2> 4548575 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4548575 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4548576 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4548578 INFO  (searcherExecutor-5651-thread-1-processing-n:127.0.0.1:56693_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@6345b93c[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4548578 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595998011247296512
   [junit4]   2> 4548583 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 4548588 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4548588 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4548588 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56693/control_collection_shard1_replica_n1/
   [junit4]   2> 4548589 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4548589 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:56693/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 4548589 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 4548593 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56693/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 4548695 INFO  (zkCallback-2673-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4548744 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4548747 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1620
   [junit4]   2> 4548752 INFO  (qtp471384468-19412) [n:127.0.0.1:56693_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 4548848 INFO  (zkCallback-2673-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4549008 INFO  (OverseerCollectionConfigSetProcessor-73519978192896004-127.0.0.1:56693_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4549752 INFO  (qtp471384468-19412) [n:127.0.0.1:56693_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56693_&wt=javabin&version=2} status=0 QTime=2754
   [junit4]   2> 4549758 INFO  (zkConnectionManagerCallback-2687-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4549759 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4549760 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42973/solr ready
   [junit4]   2> 4549760 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 4549761 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4549763 INFO  (OverseerThreadFactory-5648-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 4549764 WARN  (OverseerThreadFactory-5648-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 4549969 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 4549970 INFO  (qtp471384468-19416) [n:127.0.0.1:56693_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=208
   [junit4]   2> 4550112 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/shard-1-001 of type NRT
   [junit4]   2> 4550112 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T08:27:37+11:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 4550114 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4550114 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4550114 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 4550114 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4ed49f01{/,null,AVAILABLE}
   [junit4]   2> 4550115 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7671bd91{HTTP/1.1,[http/1.1]}{127.0.0.1:51927}
   [junit4]   2> 4550115 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.Server Started @4550168ms
   [junit4]   2> 4550115 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:41318/hdfs__localhost_41318__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=51927, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/shard-1-001/cores}
   [junit4]   2> 4550115 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4550115 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4550115 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.4.0
   [junit4]   2> 4550115 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4550115 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4550116 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-26T11:05:13.900Z
   [junit4]   2> 4550117 INFO  (zkConnectionManagerCallback-2689-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4550118 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4550118 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/shard-1-001/solr.xml
   [junit4]   2> 4550123 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4550123 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4550124 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4550137 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42973/solr
   [junit4]   2> 4550139 INFO  (zkConnectionManagerCallback-2693-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4550142 INFO  (zkConnectionManagerCallback-2695-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4550152 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4550154 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 4550157 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51927_
   [junit4]   2> 4550159 INFO  (zkCallback-2673-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4550165 INFO  (zkCallback-2686-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4550166 INFO  (zkCallback-2678-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4550170 INFO  (zkCallback-2694-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 4551635 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465
   [junit4]   2> 4551650 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465
   [junit4]   2> 4551650 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465
   [junit4]   2> 4551653 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/shard-1-001/cores
   [junit4]   2> 4551656 INFO  (zkConnectionManagerCallback-2700-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4551658 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 4551659 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [n:127.0.0.1:51927_    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42973/solr ready
   [junit4]   2> 4551695 INFO  (qtp2143335884-19475) [n:127.0.0.1:51927_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:51927_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4551697 INFO  (OverseerCollectionConfigSetProcessor-73519978192896004-127.0.0.1:56693_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4551698 INFO  (OverseerThreadFactory-5648-thread-3) [    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:51927_ for creating new replica
   [junit4]   2> 4551701 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4551701 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4552743 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 4552761 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
   [junit4]   2> 4552915 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 4552934 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
   [junit4]   2> 4552935 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23c8c465
   [junit4]   2> 4552935 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:41318/solr_hdfs_home
   [junit4]   2> 4552936 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 4552936 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4552936 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[hdfs://localhost:41318/solr_hdfs_home/collection1/core_node42/data/]
   [junit4]   2> 4552939 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:41318/solr_hdfs_home/collection1/core_node42/data/snapshot_metadata
   [junit4]   2> 4552950 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4552950 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 4552950 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 4552956 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4552956 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:41318/solr_hdfs_home/collection1/core_node42/data
   [junit4]   2> 4552974 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:41318/solr_hdfs_home/collection1/core_node42/data/index
   [junit4]   2> 4552981 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 4552981 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 4552982 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 4552986 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 4552986 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.21955043209206915]
   [junit4]   2> 4553018 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34921 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-707dc755-ade2-429f-b5a1-117531794ff4:NORMAL:127.0.0.1:33038|RBW], ReplicaUC[[DISK]DS-069a2087-b8c5-4c49-8515-76936b23b188:NORMAL:127.0.0.1:34921|RBW]]} size 0
   [junit4]   2> 4553022 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33038 is added to blk_1073741826_1002 size 69
   [junit4]   2> 4553033 WARN  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 4553098 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 4553098 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4553098 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 4553136 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4553136 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4553139 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4480062527442674]
   [junit4]   2> 4553163 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@fb59f42[collection1_shard2_replica_n41] main]
   [junit4]   2> 4553164 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4553165 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4553166 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4553166 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595998016058163200
   [junit4]   2> 4553169 INFO  (searcherExecutor-5662-thread-1-processing-n:127.0.0.1:51927_ x:collection1_shard2_replica_n41 c:collection1 s:shard2) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@fb59f42[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4553173 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node42=0}, version=0}
   [junit4]   2> 4553176 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4553176 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4553176 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51927/collection1_shard2_replica_n41/
   [junit4]   2> 4553176 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4553176 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:51927/collection1_shard2_replica_n41/ has no replicas
   [junit4]   2> 4553176 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 4553180 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:51927/collection1_shard2_replica_n41/ shard2
   [junit4]   2> 4553332 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4553334 INFO  (qtp2143335884-19481) [n:127.0.0.1:51927_ c:collection1 s:shard2  x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1633
   [junit4]   2> 4553336 INFO  (qtp2143335884-19475) [n:127.0.0.1:51927_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:51927_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1641
   [junit4]   2> 4553486 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/shard-2-001 of type NRT
   [junit4]   2> 4553487 INFO  (TEST-HdfsUnloadDistributedZkTest.test-seed#[46ACA90D8DC97E65]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T08:27:37+11:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 4553488 INFO  (TEST-HdfsUnloadDist

[...truncated too long message...]

r.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:120)
   [junit4]    > 	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:72)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 4618385 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 4618524 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 4618636 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:41318) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 4618636 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:41318) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-326446808-10.41.0.5-1522062306365 (Datanode Uuid 9568d52c-56cc-454e-ae12-28b007afbf44) service to localhost/127.0.0.1:41318
   [junit4]   2> 4618640 WARN  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 4618688 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 4618708 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:41318) [    ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
   [junit4]   2> 4618708 WARN  (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:41318) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-326446808-10.41.0.5-1522062306365 (Datanode Uuid ec847861-ef8d-4ddd-9716-fd0956c64d00) service to localhost/127.0.0.1:41318
   [junit4]   2> 4618733 INFO  (SUITE-HdfsUnloadDistributedZkTest-seed#[46ACA90D8DC97E65]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_46ACA90D8DC97E65-001
   [junit4]   2> Mar 26, 2018 11:06:22 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=BlockTreeOrds(blocksize=128), a_t=BlockTreeOrds(blocksize=128), id=PostingsFormat(name=Direct), text=Lucene50(blocksize=128)}, docValues:{other_tl1=DocValuesFormat(name=Lucene70), range_facet_l_dv=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Lucene70), id_i1=DocValuesFormat(name=Lucene70), a_i1=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Direct), range_facet_l=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=542, maxMBSortInHeap=7.579881430460217, sim=RandomSimilarity(queryNorm=false): {}, locale=mk-MK, timezone=America/Porto_Velho
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=169867320,total=487587840
   [junit4]   2> NOTE: All tests run in this JVM: [LIROnShardRestartTest, CollectionsAPISolrJTest, ChangedSchemaMergeTest, SOLR749Test, IndexSchemaTest, AssignBackwardCompatibilityTest, SolrCloudReportersTest, DocValuesMissingTest, TestRandomFlRTGCloud, TestConfigSets, SortByFunctionTest, ExplicitHLLTest, MoveReplicaHDFSFailoverTest, TestConfigOverlay, SegmentsInfoRequestHandlerTest, AutoScalingHandlerTest, ScheduledTriggerTest, TestCharFilters, SyncSliceTest, TestReplicationHandler, TestPayloadCheckQParserPlugin, SolrInfoBeanTest, TestHttpShardHandlerFactory, PathHierarchyTokenizerFactoryTest, TestSolrCloudWithDelegationTokens, FacetPivotSmallTest, ResourceLoaderTest, DirectSolrConnectionTest, TestSchemaSimilarityResource, ConnectionManagerTest, CoreAdminRequestStatusTest, SchemaVersionSpecificBehaviorTest, TestReversedWildcardFilterFactory, SolrJmxReporterCloudTest, SoftAutoCommitTest, ScriptEngineTest, ActionThrottleTest, TestRecoveryHdfs, RuleEngineTest, TestSystemIdResolver, VersionInfoTest, TestCloudSchemaless, TestCopyFieldCollectionResource, TestUniqueKeyFieldResource, CursorMarkTest, TestMultiWordSynonyms, TestLegacyNumericRangeQueryBuilder, DataDrivenBlockJoinTest, QueryParsingTest, BasicFunctionalityTest, TestCustomSort, CheckHdfsIndexTest, TestFreeTextSuggestions, SpatialRPTFieldTypeTest, TestDocBasedVersionConstraints, DeleteInactiveReplicaTest, TestRequestStatusCollectionAPI, TestOrdValues, JSONWriterTest, TestSweetSpotSimilarityFactory, TestSolrCoreParser, SaslZkACLProviderTest, SearchHandlerTest, TestStreamBody, TestAnalyzeInfixSuggestions, TestDistributedMap, DirectUpdateHandlerOptimizeTest, BadComponentTest, TermsComponentTest, TestInPlaceUpdatesDistrib, TestStressVersions, PreAnalyzedFieldManagedSchemaCloudTest, MetricsHandlerTest, BlockDirectoryTest, TestReloadAndDeleteDocs, TestFuzzyAnalyzedSuggestions, QueryEqualityTest, UpdateRequestProcessorFactoryTest, TestNodeLostTrigger, TestUtilizeNode, TestCursorMarkWithoutUniqueKey, TestPostingsSolrHighlighter, TestCoreBackup, CustomHighlightComponentTest, DeleteReplicaTest, SignatureUpdateProcessorFactoryTest, TestSha256AuthenticationProvider, AssignTest, VMParamsZkACLAndCredentialsProvidersTest, AnalyticsMergeStrategyTest, TestDFISimilarityFactory, SubstringBytesRefFilterTest, HLLUtilTest, SimplePostToolTest, TestPartialUpdateDeduplication, TestSolrCloudWithHadoopAuthPlugin, SliceStateTest, TestFastLRUCache, TestSurroundQueryParser, SolrMetricsIntegrationTest, FastVectorHighlighterTest, TestSlowCompositeReaderWrapper, TestSimpleTrackingShardHandler, DistribCursorPagingTest, TestRequestForwarding, RestartWhileUpdatingTest, PeerSyncReplicationTest, TestRTGBase, HdfsChaosMonkeySafeLeaderTest, BasicZkTest, SystemInfoHandlerTest, BasicAuthIntegrationTest, TestLuceneIndexBackCompat, OverseerRolesTest, RankQueryTest, TestSubQueryTransformerCrossCore, TestJsonFacets, TestDistributedGrouping, TestReload, BlockJoinFacetDistribTest, JsonLoaderTest, AddBlockUpdateTest, TestInfoStreamLogging, HdfsLockFactoryTest, TestFieldTypeResource, HdfsRecoveryZkTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsUnloadDistributedZkTest]
   [junit4] Completed [616/785 (1!)] on J1 in 86.05s, 1 test, 1 error <<< FAILURES!

[...truncated 9868 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid11603.hprof

Total time: 284 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 199 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/199/

1 tests failed.
FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1523615867055,generation=2,filelist=[_5j6.cfe, _5j6.cfs, _5j6.si, _5k0.doc, _5k0.fdt, _5k0.fdx, _5k0.fnm, _5k0.nvd, _5k0.nvm, _5k0.si, _5k0.tim, _5k0.tip, _5k1.doc, _5k1.fdt, _5k1.fdx, _5k1.fnm, _5k1.nvd, _5k1.nvm, _5k1.si, _5k1.tim, _5k1.tip, _5k3.doc, _5k3.fdt, _5k3.fdx, _5k3.fnm, _5k3.nvd, _5k3.nvm, _5k3.si, _5k3.tim, _5k3.tip, _5k4.doc, _5k4.fdt, _5k4.fdx, _5k4.fnm, _5k4.nvd, _5k4.nvm, _5k4.si, _5k4.tim, _5k4.tip, _5k8.cfe, _5k8.cfs, _5k8.si, _5kf.doc, _5kf.fdt, _5kf.fdx, _5kf.fnm, _5kf.nvd, _5kf.nvm, _5kf.si, _5kf.tim, _5kf.tip, _5kg.doc, _5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg.tim, _5kg.tip, _5kl.doc, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl.tim, _5kl.tip, _5ko.doc, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko.tim, _5ko.tip, _5kr.doc, _5kr.fdt, _5kr.fdx, _5kr.fnm, _5kr.nvd, _5kr.nvm, _5kr.si, _5kr.tim, _5kr.tip, _5ks.doc, _5ks.fdt, _5ks.fdx, _5ks.fnm, _5ks.nvd, _5ks.nvm, _5ks.si, _5ks.tim, _5ks.tip, _5kt.cfe, _5kt.cfs, _5kt.si, _5ku.doc, _5ku.fdt, _5ku.fdx, _5ku.fnm, _5ku.nvd, _5ku.nvm, _5ku.si, _5ku.tim, _5ku.tip, _5kv.doc, _5kv.fdt, _5kv.fdx, _5kv.fnm, _5kv.nvd, _5kv.nvm, _5kv.si, _5kv.tim, _5kv.tip, _5m9.doc, _5m9.fdt, _5m9.fdx, _5m9.fnm, _5m9.nvd, _5m9.nvm, _5m9.si, _5m9.tim, _5m9.tip, _5mb.cfe, _5mb.cfs, _5mb.si, _5mc.doc, _5mc.fdt, _5mc.fdx, _5mc.fnm, _5mc.nvd, _5mc.nvm, _5mc.si, _5mc.tim, _5mc.tip, _5md.doc, _5md.fdt, _5md.fdx, _5md.fnm, _5md.nvd, _5md.nvm, _5md.si, _5md.tim, _5md.tip, _5me.doc, _5me.fdt, _5me.fdx, _5me.fnm, _5me.nvd, _5me.nvm, _5me.si, _5me.tim, _5me.tip, _5mf.doc, _5mf.fdt, _5mf.fdx, _5mf.fnm, _5mf.nvd, _5mf.nvm, _5mf.si, _5mf.tim, _5mf.tip, _5mg.doc, _5mg.fdt, _5mg.fdx, _5mg.fnm, _5mg.nvd, _5mg.nvm, _5mg.si, _5mg.tim, _5mg.tip, segments_2]}]> but was:<[{indexVersion=1523615867055,generation=2,filelist=[_5j6.cfe, _5j6.cfs, _5j6.si, _5k0.doc, _5k0.fdt, _5k0.fdx, _5k0.fnm, _5k0.nvd, _5k0.nvm, _5k0.si, _5k0.tim, _5k0.tip, _5k1.doc, _5k1.fdt, _5k1.fdx, _5k1.fnm, _5k1.nvd, _5k1.nvm, _5k1.si, _5k1.tim, _5k1.tip, _5k3.doc, _5k3.fdt, _5k3.fdx, _5k3.fnm, _5k3.nvd, _5k3.nvm, _5k3.si, _5k3.tim, _5k3.tip, _5k4.doc, _5k4.fdt, _5k4.fdx, _5k4.fnm, _5k4.nvd, _5k4.nvm, _5k4.si, _5k4.tim, _5k4.tip, _5k8.cfe, _5k8.cfs, _5k8.si, _5kf.doc, _5kf.fdt, _5kf.fdx, _5kf.fnm, _5kf.nvd, _5kf.nvm, _5kf.si, _5kf.tim, _5kf.tip, _5kg.doc, _5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg.tim, _5kg.tip, _5kl.doc, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl.tim, _5kl.tip, _5ko.doc, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko.tim, _5ko.tip, _5kr.doc, _5kr.fdt, _5kr.fdx, _5kr.fnm, _5kr.nvd, _5kr.nvm, _5kr.si, _5kr.tim, _5kr.tip, _5ks.doc, _5ks.fdt, _5ks.fdx, _5ks.fnm, _5ks.nvd, _5ks.nvm, _5ks.si, _5ks.tim, _5ks.tip, _5kt.cfe, _5kt.cfs, _5kt.si, _5ku.doc, _5ku.fdt, _5ku.fdx, _5ku.fnm, _5ku.nvd, _5ku.nvm, _5ku.si, _5ku.tim, _5ku.tip, _5kv.doc, _5kv.fdt, _5kv.fdx, _5kv.fnm, _5kv.nvd, _5kv.nvm, _5kv.si, _5kv.tim, _5kv.tip, _5m9.doc, _5m9.fdt, _5m9.fdx, _5m9.fnm, _5m9.nvd, _5m9.nvm, _5m9.si, _5m9.tim, _5m9.tip, _5mb.cfe, _5mb.cfs, _5mb.si, _5mc.doc, _5mc.fdt, _5mc.fdx, _5mc.fnm, _5mc.nvd, _5mc.nvm, _5mc.si, _5mc.tim, _5mc.tip, _5md.doc, _5md.fdt, _5md.fdx, _5md.fnm, _5md.nvd, _5md.nvm, _5md.si, _5md.tim, _5md.tip, _5me.doc, _5me.fdt, _5me.fdx, _5me.fnm, _5me.nvd, _5me.nvm, _5me.si, _5me.tim, _5me.tip, _5mf.doc, _5mf.fdt, _5mf.fdx, _5mf.fnm, _5mf.nvd, _5mf.nvm, _5mf.si, _5mf.tim, _5mf.tip, _5mg.doc, _5mg.fdt, _5mg.fdx, _5mg.fnm, _5mg.nvd, _5mg.nvm, _5mg.si, _5mg.tim, _5mg.tip, segments_2]}, {indexVersion=1523615867055,generation=3,filelist=[_5kw.cfe, _5kw.cfs, _5kw.si, _5m9.doc, _5m9.fdt, _5m9.fdx, _5m9.fnm, _5m9.nvd, _5m9.nvm, _5m9.si, _5m9.tim, _5m9.tip, _5mb.cfe, _5mb.cfs, _5mb.si, _5mc.doc, _5mc.fdt, _5mc.fdx, _5mc.fnm, _5mc.nvd, _5mc.nvm, _5mc.si, _5mc.tim, _5mc.tip, _5md.doc, _5md.fdt, _5md.fdx, _5md.fnm, _5md.nvd, _5md.nvm, _5md.si, _5md.tim, _5md.tip, _5me.doc, _5me.fdt, _5me.fdx, _5me.fnm, _5me.nvd, _5me.nvm, _5me.si, _5me.tim, _5me.tip, _5mf.doc, _5mf.fdt, _5mf.fdx, _5mf.fnm, _5mf.nvd, _5mf.nvm, _5mf.si, _5mf.tim, _5mf.tip, _5mg.doc, _5mg.fdt, _5mg.fdx, _5mg.fnm, _5mg.nvd, _5mg.nvm, _5mg.si, _5mg.tim, _5mg.tip, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1523615867055,generation=2,filelist=[_5j6.cfe, _5j6.cfs, _5j6.si, _5k0.doc, _5k0.fdt, _5k0.fdx, _5k0.fnm, _5k0.nvd, _5k0.nvm, _5k0.si, _5k0.tim, _5k0.tip, _5k1.doc, _5k1.fdt, _5k1.fdx, _5k1.fnm, _5k1.nvd, _5k1.nvm, _5k1.si, _5k1.tim, _5k1.tip, _5k3.doc, _5k3.fdt, _5k3.fdx, _5k3.fnm, _5k3.nvd, _5k3.nvm, _5k3.si, _5k3.tim, _5k3.tip, _5k4.doc, _5k4.fdt, _5k4.fdx, _5k4.fnm, _5k4.nvd, _5k4.nvm, _5k4.si, _5k4.tim, _5k4.tip, _5k8.cfe, _5k8.cfs, _5k8.si, _5kf.doc, _5kf.fdt, _5kf.fdx, _5kf.fnm, _5kf.nvd, _5kf.nvm, _5kf.si, _5kf.tim, _5kf.tip, _5kg.doc, _5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg.tim, _5kg.tip, _5kl.doc, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl.tim, _5kl.tip, _5ko.doc, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko.tim, _5ko.tip, _5kr.doc, _5kr.fdt, _5kr.fdx, _5kr.fnm, _5kr.nvd, _5kr.nvm, _5kr.si, _5kr.tim, _5kr.tip, _5ks.doc, _5ks.fdt, _5ks.fdx, _5ks.fnm, _5ks.nvd, _5ks.nvm, _5ks.si, _5ks.tim, _5ks.tip, _5kt.cfe, _5kt.cfs, _5kt.si, _5ku.doc, _5ku.fdt, _5ku.fdx, _5ku.fnm, _5ku.nvd, _5ku.nvm, _5ku.si, _5ku.tim, _5ku.tip, _5kv.doc, _5kv.fdt, _5kv.fdx, _5kv.fnm, _5kv.nvd, _5kv.nvm, _5kv.si, _5kv.tim, _5kv.tip, _5m9.doc, _5m9.fdt, _5m9.fdx, _5m9.fnm, _5m9.nvd, _5m9.nvm, _5m9.si, _5m9.tim, _5m9.tip, _5mb.cfe, _5mb.cfs, _5mb.si, _5mc.doc, _5mc.fdt, _5mc.fdx, _5mc.fnm, _5mc.nvd, _5mc.nvm, _5mc.si, _5mc.tim, _5mc.tip, _5md.doc, _5md.fdt, _5md.fdx, _5md.fnm, _5md.nvd, _5md.nvm, _5md.si, _5md.tim, _5md.tip, _5me.doc, _5me.fdt, _5me.fdx, _5me.fnm, _5me.nvd, _5me.nvm, _5me.si, _5me.tim, _5me.tip, _5mf.doc, _5mf.fdt, _5mf.fdx, _5mf.fnm, _5mf.nvd, _5mf.nvm, _5mf.si, _5mf.tim, _5mf.tip, _5mg.doc, _5mg.fdt, _5mg.fdx, _5mg.fnm, _5mg.nvd, _5mg.nvm, _5mg.si, _5mg.tim, _5mg.tip, segments_2]}]> but was:<[{indexVersion=1523615867055,generation=2,filelist=[_5j6.cfe, _5j6.cfs, _5j6.si, _5k0.doc, _5k0.fdt, _5k0.fdx, _5k0.fnm, _5k0.nvd, _5k0.nvm, _5k0.si, _5k0.tim, _5k0.tip, _5k1.doc, _5k1.fdt, _5k1.fdx, _5k1.fnm, _5k1.nvd, _5k1.nvm, _5k1.si, _5k1.tim, _5k1.tip, _5k3.doc, _5k3.fdt, _5k3.fdx, _5k3.fnm, _5k3.nvd, _5k3.nvm, _5k3.si, _5k3.tim, _5k3.tip, _5k4.doc, _5k4.fdt, _5k4.fdx, _5k4.fnm, _5k4.nvd, _5k4.nvm, _5k4.si, _5k4.tim, _5k4.tip, _5k8.cfe, _5k8.cfs, _5k8.si, _5kf.doc, _5kf.fdt, _5kf.fdx, _5kf.fnm, _5kf.nvd, _5kf.nvm, _5kf.si, _5kf.tim, _5kf.tip, _5kg.doc, _5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg.tim, _5kg.tip, _5kl.doc, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl.tim, _5kl.tip, _5ko.doc, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko.tim, _5ko.tip, _5kr.doc, _5kr.fdt, _5kr.fdx, _5kr.fnm, _5kr.nvd, _5kr.nvm, _5kr.si, _5kr.tim, _5kr.tip, _5ks.doc, _5ks.fdt, _5ks.fdx, _5ks.fnm, _5ks.nvd, _5ks.nvm, _5ks.si, _5ks.tim, _5ks.tip, _5kt.cfe, _5kt.cfs, _5kt.si, _5ku.doc, _5ku.fdt, _5ku.fdx, _5ku.fnm, _5ku.nvd, _5ku.nvm, _5ku.si, _5ku.tim, _5ku.tip, _5kv.doc, _5kv.fdt, _5kv.fdx, _5kv.fnm, _5kv.nvd, _5kv.nvm, _5kv.si, _5kv.tim, _5kv.tip, _5m9.doc, _5m9.fdt, _5m9.fdx, _5m9.fnm, _5m9.nvd, _5m9.nvm, _5m9.si, _5m9.tim, _5m9.tip, _5mb.cfe, _5mb.cfs, _5mb.si, _5mc.doc, _5mc.fdt, _5mc.fdx, _5mc.fnm, _5mc.nvd, _5mc.nvm, _5mc.si, _5mc.tim, _5mc.tip, _5md.doc, _5md.fdt, _5md.fdx, _5md.fnm, _5md.nvd, _5md.nvm, _5md.si, _5md.tim, _5md.tip, _5me.doc, _5me.fdt, _5me.fdx, _5me.fnm, _5me.nvd, _5me.nvm, _5me.si, _5me.tim, _5me.tip, _5mf.doc, _5mf.fdt, _5mf.fdx, _5mf.fnm, _5mf.nvd, _5mf.nvm, _5mf.si, _5mf.tim, _5mf.tip, _5mg.doc, _5mg.fdt, _5mg.fdx, _5mg.fnm, _5mg.nvd, _5mg.nvm, _5mg.si, _5mg.tim, _5mg.tip, segments_2]}, {indexVersion=1523615867055,generation=3,filelist=[_5kw.cfe, _5kw.cfs, _5kw.si, _5m9.doc, _5m9.fdt, _5m9.fdx, _5m9.fnm, _5m9.nvd, _5m9.nvm, _5m9.si, _5m9.tim, _5m9.tip, _5mb.cfe, _5mb.cfs, _5mb.si, _5mc.doc, _5mc.fdt, _5mc.fdx, _5mc.fnm, _5mc.nvd, _5mc.nvm, _5mc.si, _5mc.tim, _5mc.tip, _5md.doc, _5md.fdt, _5md.fdx, _5md.fnm, _5md.nvd, _5md.nvm, _5md.si, _5md.tim, _5md.tip, _5me.doc, _5me.fdt, _5me.fdx, _5me.fnm, _5me.nvd, _5me.nvm, _5me.si, _5me.tim, _5me.tip, _5mf.doc, _5mf.fdt, _5mf.fdx, _5mf.fnm, _5mf.nvd, _5mf.nvm, _5mf.si, _5mf.tim, _5mf.tip, _5mg.doc, _5mg.fdt, _5mg.fdx, _5mg.fnm, _5mg.nvd, _5mg.nvm, _5mg.si, _5mg.tim, _5mg.tip, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([DAFFDC8E84BA5E7B:FF28C7BEF4F25078]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1291)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14289 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/init-core-data-001
   [junit4]   2> 4076898 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T11:27:37+14:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 4076899 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4076899 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4076899 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 4076899 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e0fa773{/solr,null,AVAILABLE}
   [junit4]   2> 4076900 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@46b6d908{HTTP/1.1,[http/1.1]}{127.0.0.1:46040}
   [junit4]   2> 4076901 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.Server Started @4077041ms
   [junit4]   2> 4076901 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=46040}
   [junit4]   2> 4076901 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4076901 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4076901 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 4076901 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 4076901 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4076901 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-13T10:34:13.147Z
   [junit4]   2> 4076901 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/solr.xml
   [junit4]   2> 4076925 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4076925 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4076926 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4077041 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4077062 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4077062 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4077064 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/.
   [junit4]   2> 4077064 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 4077065 INFO  (coreLoadExecutor-4717-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4077111 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 4077139 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 4077141 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 4077170 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/./collection1, trusted=true
   [junit4]   2> 4077171 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4077171 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4077171 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 4077173 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=90.7744140625, floorSegmentMB=1.939453125, forceMergeDeletesPctAllowed=16.205093016958976, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5398778681375698
   [junit4]   2> 4077314 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4077314 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4077314 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@61de7b48
   [junit4]   2> 4077315 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@20b0938[collection1] main]
   [junit4]   2> 4077315 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/collection1/conf
   [junit4]   2> 4077316 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 4077316 INFO  (coreLoadExecutor-4717-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4077316 INFO  (searcherExecutor-4718-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@20b0938[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4077320 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T11:27:37+14:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 4077321 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4077321 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4077321 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 4077322 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2a9a7840{/solr,null,AVAILABLE}
   [junit4]   2> 4077322 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5d7fcdab{HTTP/1.1,[http/1.1]}{127.0.0.1:46766}
   [junit4]   2> 4077322 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.Server Started @4077463ms
   [junit4]   2> 4077322 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=46766}
   [junit4]   2> 4077322 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4077322 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4077323 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 4077323 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 4077323 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4077323 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-13T10:34:13.569Z
   [junit4]   2> 4077323 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-002/solr.xml
   [junit4]   2> 4077343 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4077343 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4077344 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4077448 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4077474 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4077474 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4077477 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-002/.
   [junit4]   2> 4077477 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 4077477 INFO  (coreLoadExecutor-4727-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4077548 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 4077591 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 4077592 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 4077592 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-002/./collection1, trusted=true
   [junit4]   2> 4077593 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4077593 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4077593 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-002/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 4077595 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=90.7744140625, floorSegmentMB=1.939453125, forceMergeDeletesPctAllowed=16.205093016958976, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5398778681375698
   [junit4]   2> 4077710 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4077710 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4077719 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@69cccdf2
   [junit4]   2> 4077720 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@450ebbf7[collection1] main]
   [junit4]   2> 4077720 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-002/collection1/conf
   [junit4]   2> 4077722 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
   [junit4]   2> 4077722 INFO  (coreLoadExecutor-4727-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4077722 INFO  (searcherExecutor-4728-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@450ebbf7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4077725 INFO  (qtp1478977714-65530) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 4077725 INFO  (indexFetcher-4732-thread-1) [    ] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 4077725 INFO  (indexFetcher-4732-thread-1) [    ] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 4077725 INFO  (indexFetcher-4732-thread-1) [    ] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 4077725 INFO  (indexFetcher-4732-thread-1) [    ] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 4077725 INFO  (indexFetcher-4732-thread-1) [    ] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 4077725 INFO  (indexFetcher-4732-thread-1) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3de5bfe0[collection1] main]
   [junit4]   2> 4077726 INFO  (searcherExecutor-4728-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3de5bfe0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4078240 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@5d7fcdab{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 4078241 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=150096332
   [junit4]   2> 4078241 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 4078241 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1910bac: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@7491b704
   [junit4]   2> 4078246 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 4078246 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@68bb3da9: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@e1ceb41
   [junit4]   2> 4078258 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 4078258 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@69450003: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@f60b336
   [junit4]   2> 4078271 INFO  (coreCloseExecutor-4733-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@334cfeea
   [junit4]   2> 4078271 INFO  (coreCloseExecutor-4733-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=334cfeea
   [junit4]   2> 4078272 INFO  (coreCloseExecutor-4733-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@17a607eb: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@ea7ecd2
   [junit4]   2> 4078297 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2a9a7840{/solr,null,UNAVAILABLE}
   [junit4]   2> 4078297 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 4078300 INFO  (qtp1478977714-65536) [    x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 4078300 INFO  (qtp1478977714-65536) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:*} 0 0
   [junit4]   2> 4078312 INFO  (qtp1478977714-65535) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 4078312 INFO  (qtp1478977714-65535) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1deb5979 commitCommandVersion:0
   [junit4]   2> 4078313 INFO  (qtp1478977714-65535) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5211be8f[collection1] main]
   [junit4]   2> 4078314 INFO  (qtp1478977714-65535) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 4078314 INFO  (searcherExecutor-4718-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5211be8f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4078314 INFO  (qtp1478977714-65535) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 2
   [junit4]   2> 4078316 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@46b6d908{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 4078316 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1047449243
   [junit4]   2> 4078317 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 4078317 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@642d9732: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@1e33951f
   [junit4]   2> 4078318 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 4078318 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@59dffe37: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@e1ceb41
   [junit4]   2> 4078319 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 4078319 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@69613d9b: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@f60b336
   [junit4]   2> 4078328 INFO  (coreCloseExecutor-4735-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@100d4370
   [junit4]   2> 4078329 INFO  (coreCloseExecutor-4735-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=100d4370
   [junit4]   2> 4078329 INFO  (coreCloseExecutor-4735-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1a4799d2: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@67bd21d8
   [junit4]   2> 4078331 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5e0fa773{/solr,null,UNAVAILABLE}
   [junit4]   2> 4078331 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 4078353 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T11:27:37+14:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 4078354 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4078354 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4078365 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 4078381 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ada5734{/solr,null,AVAILABLE}
   [junit4]   2> 4078382 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@382b3c78{HTTP/1.1,[http/1.1]}{127.0.0.1:37742}
   [junit4]   2> 4078382 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.e.j.s.Server Started @4078523ms
   [junit4]   2> 4078382 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=37742}
   [junit4]   2> 4078382 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4078382 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4078430 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 4078430 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 4078430 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 4078430 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-13T10:34:14.676Z
   [junit4]   2> 4078431 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/solr.xml
   [junit4]   2> 4078435 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 4078435 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 4078436 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4078526 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4078542 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4078542 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4078552 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/.
   [junit4]   2> 4078552 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartupWithNoActivity-seed#[DAFFDC8E84BA5E7B]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 4078553 INFO  (coreLoadExecutor-4742-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4078575 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 4078603 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 4078604 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 4078604 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/./collection1, trusted=true
   [junit4]   2> 4078605 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42de8a09
   [junit4]   2> 4078605 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 4078605 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 4078607 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4300cf80
   [junit4]   2> 4078736 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4078736 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4078737 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=8, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.26060502327489976]
   [junit4]   2> 4078738 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@9e49334[collection1] main]
   [junit4]   2> 4078738 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_DAFFDC8E84BA5E7B-001/solr-instance-001/collection1/conf
   [junit4]   2> 4078739 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml
   [junit4]   2> 4078739 INFO  (coreLoadExecutor-4742-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 4078739 INFO  (searcherExecutor-4743-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@9e49334[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4078752 INFO  (qtp943432979-65572) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 0
   [junit4]   2> 4078753 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1]} 0 0
   [junit4]   2> 4078768 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 0
   [junit4]   2> 4078771 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[3]} 0 0
   [junit4]   2> 4078772 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[4]} 0 0
   [junit4]   2> 4078772 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[5]} 0 0
   [junit4]   2> 4078772 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[6]} 0 0
   [junit4]   2> 4078773 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[7]} 0 0
   [junit4]   2> 4078773 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[8]} 0 0
   [junit4]   2> 4078773 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[9]} 0 0
   [junit4]   2> 4078774 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[10]} 0 0
   [junit4]   2> 4078774 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[11]} 0 0
   [junit4]   2> 4078774 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[12]} 0 0
   [junit4]   2> 4078775 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[13]} 0 0
   [junit4]   2> 4078775 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[14]} 0 0
   [junit4]   2> 4078788 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[15]} 0 0
   [junit4]   2> 4078788 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[16]} 0 0
   [junit4]   2> 4078788 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[17]} 0 0
   [junit4]   2> 4078789 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[18]} 0 0
   [junit4]   2> 4078789 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[19]} 0 0
   [junit4]   2> 4078789 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[20]} 0 0
   [junit4]   2> 4078790 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[21]} 0 0
   [junit4]   2> 4078790 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[22]} 0 0
   [junit4]   2> 4078790 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[23]} 0 0
   [junit4]   2> 4078791 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[24]} 0 0
   [junit4]   2> 4078791 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[25]} 0 0
   [junit4]   2> 4078791 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[26]} 0 0
   [junit4]   2> 4078792 INFO  (qtp943432979-65576) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[27]} 0 0
   [junit4]   2> 4078792 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[28]} 0 0
   [junit4]   2> 4078792 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[29]} 0 0
   [junit4]   2> 4078793 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[30]} 0 0
   [junit4]   2> 4078793 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[31]} 0 0
   [junit4]   2> 4078794 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[32]} 0 0
   [junit4]   2> 4078796 INFO  (qtp943432979-65577) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[33]} 0 1
   [junit4]   2> 4078833 INFO  (qtp943432979-65580) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[34]} 0 29
   [junit4]   2> 4078834 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[35]} 0 0
   [junit4]   2> 4078834 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[36]} 0 0
   [junit4]   2> 4078835 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[37]} 0 0
   [junit4]   2> 4078835 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[38]} 0 0
   [junit4]   2> 4078835 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[39]} 0 0
   [junit4]   2> 4078836 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[40]} 0 0
   [junit4]   2> 4078836 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[41]} 0 0
   [junit4]   2> 4078836 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[42]} 0 0
   [junit4]   2> 4078837 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[43]} 0 0
   [junit4]   2> 4078837 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[44]} 0 0
   [junit4]   2> 4078837 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[45]} 0 0
   [junit4]   2> 4078838 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[46]} 0 0
   [junit4]   2> 4078838 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[47]} 0 0
   [junit4]   2> 4078838 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[48]} 0 0
   [junit4]   2> 4078838 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[49]} 0 0
   [junit4]   2> 4078839 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[50]} 0 0
   [junit4]   2> 4078839 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[51]} 0 0
   [junit4]   2> 4078839 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[52]} 0 0
   [junit4]   2> 4078840 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[53]} 0 0
   [junit4]   2> 4078840 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[54]} 0 0
   [junit4]   2> 4078861 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[55]} 0 0
   [junit4]   2> 4078861 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[56]} 0 0
   [junit4]   2> 4078861 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[57]} 0 0
   [junit4]   2> 4078862 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[58]} 0 0
   [junit4]   2> 4078862 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[59]} 0 0
   [junit4]   2> 4078862 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[60]} 0 0
   [junit4]   2> 4078863 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[61]} 0 0
   [junit4]   2> 4078863 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[62]} 0 0
   [junit4]   2> 4078863 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[63]} 0 0
   [junit4]   2> 4078864 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[64]} 0 0
   [junit4]   2> 4078864 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[65]} 0 0
   [junit4]   2> 4078864 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[66]} 0 0
   [junit4]   2> 4078877 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[67]} 0 13
   [junit4]   2> 4078878 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[68]} 0 0
   [junit4]   2> 4078878 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[69]} 0 0
   [junit4]   2> 4078878 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[70]} 0 0
   [junit4]   2> 4078879 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[71]} 0 0
   [junit4]   2> 4078879 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[72]} 0 0
   [junit4]   2> 4078879 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[73]} 0 0
   [junit4]   2> 4078880 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[74]} 0 0
   [junit4]   2> 4078880 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[75]} 0 0
   [junit4]   2> 4078880 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[76]} 0 0
   [junit4]   2> 4078881 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[77]} 0 0
   [junit4]   2> 4078897 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[78]} 0 0
   [junit4]   2> 4078898 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[79]} 0 0
   [junit4]   2> 4078898 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[80]} 0 0
   [junit4]   2> 4078898 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[81]} 0 0
   [junit4]   2> 4078899 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[82]} 0 0
   [junit4]   2> 4078899 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[83]} 0 0
   [junit4]   2> 4078899 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[84]} 0 0
   [junit4]   2> 4078900 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[85]} 0 0
   [junit4]   2> 4078900 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[86]} 0 0
   [junit4]   2> 4078900 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[87]} 0 0
   [junit4]   2> 4078901 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[88]} 0 0
   [junit4]   2> 4078913 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[89]} 0 12
   [junit4]   2> 4078914 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[90]} 0 0
   [junit4]   2> 4078914 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[91]} 0 0
   [junit4]   2> 4078914 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[92]} 0 0
   [junit4]   2> 4078914 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[93]} 0 0
   [junit4]   2> 4078915 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[94]} 0 0
   [junit4]   2> 4078915 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[95]} 0 0
   [junit4]   2> 4078915 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[96]} 0 0
   [junit4]   2> 4078916 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[97]} 0 0
   [junit4]   2> 4078916 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[98]} 0 0
   [junit4]   2> 4078916 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[99]} 0 0
   [junit4]   2> 4078917 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[100]} 0 0
   [junit4]   2> 4078934 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[101]} 0 17
   [junit4]   2> 4078934 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[102]} 0 0
   [junit4]   2> 4078935 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[103]} 0 0
   [junit4]   2> 4078935 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[104]} 0 0
   [junit4]   2> 4078935 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[105]} 0 0
   [junit4]   2> 4078936 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[106]} 0 0
   [junit4]   2> 4078936 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[107]} 0 0
   [junit4]   2> 4078936 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[108]} 0 0
   [junit4]   2> 4078936 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[109]} 0 0
   [junit4]   2> 4078937 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[110]} 0 0
   [junit4]   2> 4078937 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[111]} 0 0
   [junit4]   2> 4078974 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[112]} 0 37
   [junit4]   2> 4078975 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[113]} 0 0
   [junit4]   2> 4078975 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[114]} 0 0
   [junit4]   2> 4078975 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[115]} 0 0
   [junit4]   2> 4078976 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[116]} 0 0
   [junit4]   2> 4078976 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[117]} 0 0
   [junit4]   2> 4078976 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[118]} 0 0
   [junit4]   2> 4078976 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[119]} 0 0
   [junit4]   2> 4078995 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[120]} 0 17
   [junit4]   2> 4078995 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[121]} 0 0
   [junit4]   2> 4078995 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[122]} 0 0
   [junit4]   2> 4078996 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[123]} 0 0
   [junit4]   2> 4078996 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[124]} 0 0
   [junit4]   2> 4078996 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[125]} 0 0
   [junit4]   2> 4078997 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[126]} 0 0
   [junit4]   2> 4078997 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[127]} 0 0
   [junit4]   2> 4078997 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[128]} 0 0
   [junit4]   2> 4078998 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[129]} 0 0
   [junit4]   2> 4078998 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[130]} 0 0
   [junit4]   2> 4078998 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[131]} 0 0
   [junit4]   2> 4079031 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[132]} 0 32
   [junit4]   2> 4079032 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[133]} 0 0
   [junit4]   2> 4079032 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[134]} 0 0
   [junit4]   2> 4079033 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[135]} 0 0
   [junit4]   2> 4079033 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[136]} 0 0
   [junit4]   2> 4079034 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[137]} 0 0
   [junit4]   2> 4079034 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[138]} 0 0
   [junit4]   2> 4079034 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[139]} 0 0
   [junit4]   2> 4079035 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[140]} 0 0
   [junit4]   2> 4079035 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[141]} 0 0
   [junit4]   2> 4079047 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[142]} 0 0
   [junit4]   2> 4079048 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[143]} 0 0
   [junit4]   2> 4079051 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[144]} 0 0
   [junit4]   2> 4079068 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[145]} 0 0
   [junit4]   2> 4079068 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[146]} 0 0
   [junit4]   2> 4079069 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[147]} 0 0
   [junit4]   2> 4079069 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[148]} 0 0
   [junit4]   2> 4079069 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[149]} 0 0
   [junit4]   2> 4079070 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[150]} 0 0
   [junit4]   2> 4079070 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[151]} 0 0
   [junit4]   2> 4079070 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[152]} 0 0
   [junit4]   2> 4079070 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[153]} 0 0
   [junit4]   2> 4079071 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[154]} 0 0
   [junit4]   2> 4079071 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[155]} 0 0
   [junit4]   2> 4079071 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[156]} 0 0
   [junit4]   2> 4079072 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[157]} 0 0
   [junit4]   2> 4079084 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[158]} 0 0
   [junit4]   2> 4079085 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[159]} 0 0
   [junit4]   2> 4079085 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[160]} 0 0
   [junit4]   2> 4079085 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[161]} 0 0
   [junit4]   2> 4079086 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[162]} 0 0
   [junit4]   2> 4079086 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[163]} 0 0
   [junit4]   2> 4079086 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[164]} 0 0
   [junit4]   2> 4079087 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[165]} 0 0
   [junit4]   2> 4079087 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[166]} 0 0
   [junit4]   2> 4079087 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[167]} 0 0
   [junit4]   2> 4079088 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[168]} 0 0
   [junit4]   2> 4079105 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[169]} 0 17
   [junit4]   2> 4079105 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[170]} 0 0
   [junit4]   2> 4079106 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[171]} 0 0
   [junit4]   2> 4079106 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[172]} 0 0
   [junit4]   2> 4079106 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[173]} 0 0
   [junit4]   2> 4079107 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[174]} 0 0
   [junit4]   2> 4079107 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[175]} 0 0
   [junit4]   2> 4079107 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[176]} 0 0
   [junit4]   2> 4079108 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[177]} 0 0
   [junit4]   2> 4079108 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[178]} 0 0
   [junit4]   2> 4079108 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[179]} 0 0
   [junit4]   2> 4079121 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[180]} 0 0
   [junit4]   2> 4079121 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[181]} 0 0
   [junit4]   2> 4079121 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[182]} 0 0
   [junit4]   2> 4079122 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[183]} 0 0
   [junit4]   2> 4079122 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[184]} 0 0
   [junit4]   2> 4079122 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[185]} 0 0
   [junit4]   2> 4079122 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[186]} 0 0
   [junit4]   2> 4079123 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[187]} 0 0
   [junit4]   2> 4079123 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[188]} 0 0
   [junit4]   2> 4079123 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[189]} 0 0
   [junit4]   2> 4079124 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[190]} 0 0
   [junit4]   2> 4079124 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[191]} 0 0
   [junit4]   2> 4079124 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[192]} 0 0
   [junit4]   2> 4079124 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[193]} 0 0
   [junit4]   2> 4079141 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[194]} 0 0
   [junit4]   2> 4079142 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[195]} 0 0
   [junit4]   2> 4079142 INFO  (qtp943432979-65578) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[196]} 0 0
   [junit4]   2> 4079142 INFO  (qtp943432979-65583) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[197]} 0 0
   [ju

[...truncated too long message...]

y-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1567657602
     [copy] Copying 40 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1567657602
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1567657602

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1964697925
     [copy] Copying 246 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1964697925
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1964697925

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 26ms :: artifacts dl 1ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 481 minutes 14 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 198 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/198/

3 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test

Error Message:
Error from server at https://127.0.0.1:40759/solr: Could not restore core

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:40759/solr: Could not restore core
	at __randomizedtesting.SeedInfo.seed([93925E63CC51A4B4:1BC661B962ADC94C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:287)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:142)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTrigger

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
	at __randomizedtesting.SeedInfo.seed([93925E63CC51A4B4:B5F8E05AD75E0BD8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTrigger(TestTriggerIntegration.java:408)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([93925E63CC51A4B4:A02076A7C1E67E03]:0)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:424)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:129)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:113)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:292)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:372)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:113)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:78)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:197)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4628)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2335)
	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3570)
	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3522)
	at org.apache.lucene.index.RandomIndexWriter.maybeFlushOrCommit(RandomIndexWriter.java:216)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:188)
	at org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit(TestDocTermOrds.java:171)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)




Build Log:
[...truncated 12986 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_93925E63CC51A4B4-001/init-core-data-001
   [junit4]   2> 172481 WARN  (SUITE-TestTriggerIntegration-seed#[93925E63CC51A4B4]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 172482 INFO  (SUITE-TestTriggerIntegration-seed#[93925E63CC51A4B4]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 172516 INFO  (SUITE-TestTriggerIntegration-seed#[93925E63CC51A4B4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 172533 INFO  (SUITE-TestTriggerIntegration-seed#[93925E63CC51A4B4]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 172791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 172861 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 172863 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 172863 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4] IGNOR/A 0.08s J2 | TestTriggerIntegration.testSearchRate
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 172970 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 173009 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 173009 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 173043 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
   [junit4]   2> 173043 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 173066 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 173067 DEBUG (ScheduledTrigger-107-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173079 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 173174 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 173174 DEBUG (ScheduledTrigger-107-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173175 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 173175 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 173175 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 173189 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 173189 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 173202 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 173202 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 173203 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 173203 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 173222 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 173223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 173223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 173223 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173294 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173332 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173384 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173409 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173444 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173482 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173517 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173554 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173590 DEBUG (simCloudManagerPool-109-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 173590 DEBUG (simCloudManagerPool-109-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 173590 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 173605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 173605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 173643 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173663 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173700 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 173725 DEBUG (simCloudManagerPool-109-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 173725 DEBUG (simCloudManagerPool-109-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 173733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 173733 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 173733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 173733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 173734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 173737 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 173737 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 173737 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 173737 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10002_solr at time 5807910489118412
   [junit4]   2> 173773 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 173773 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 173773 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 173774 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10002_solr] added at times [5807910489118412], now=5807912291932112
   [junit4]   2> 173776 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"14a243899756ccTb6dcrwj17bt0ghdhsu6uyamuw",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":5807910489118412,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5807910489118412],
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 174066 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger]
   [junit4]   2> 174191 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
   [junit4]   2> 174192 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 174192 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 174192 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 174192 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 174192 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		3
   [junit4]   2> 174192 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	3
   [junit4]   2> 174192 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		0
   [junit4]   2> 174193 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 174193 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 174193 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 174193 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 174194 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 174194 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        2
   [junit4]   2> 174194 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             1
   [junit4]   2> 174194 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 174214 DEBUG (ScheduledTrigger-110-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10002_solr
   [junit4]   2> 174227 DEBUG (AutoscalingActionExecutor-111-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14a243899756ccTb6dcrwj17bt0ghdhsu6uyamuw",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":5807910489118412,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5807910489118412],
   [junit4]   2>     "_enqueue_time_":5807926959279312,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 174250 DEBUG (AutoscalingActionExecutor-111-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 174250 DEBUG (AutoscalingActionExecutor-111-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 174250 DEBUG (AutoscalingActionExecutor-111-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 1777 ms for event id=14a243899756ccTb6dcrwj17bt0ghdhsu6uyamuw
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeAddedTrigger -Dtests.seed=93925E63CC51A4B4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca-ES -Dtests.timezone=Pacific/Samoa -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 1.33s J2 | TestTriggerIntegration.testNodeAddedTrigger <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The trigger did not fire at all
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([93925E63CC51A4B4:B5F8E05AD75E0BD8]:0)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTrigger(TestTriggerIntegration.java:408)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testCooldown
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testNodeLostTrigger
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 174394 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 174520 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 174540 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 174576 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 174812 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 174813 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 174813 DEBUG (ScheduledTrigger-110-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 174885 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 174885 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 174885 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 174906 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 174907 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 174907 DEBUG (ScheduledTrigger-110-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 174994 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 174994 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 174995 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 175141 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 175141 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 175141 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 175562 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 175563 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 175563 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 175598 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 175599 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 175599 DEBUG (ScheduledTrigger-110-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 175600 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
   [junit4]   2> 175600 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 175601 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 175601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 175601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 175601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 175601 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 175602 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 175602 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 175602 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 175673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 175673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 175711 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 175711 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 175727 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 175753 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 175753 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 175893 DEBUG (ScheduledTrigger-113-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 175907 DEBUG (simCloudManagerPool-112-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 175908 DEBUG (simCloudManagerPool-112-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 176192 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 176193 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 176227 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 176228 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 176228 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 176228 DEBUG (simCloudManagerPool-112-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 176244 DEBUG (simCloudManagerPool-112-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
   [junit4]   2> 176245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 176245 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 176245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 176245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 176245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 176245 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 176245 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 176264 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 176266 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 176266 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 176300 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 176300 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 176300 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 176337 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 176337 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 176337 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 176338 DEBUG (simCloudManagerPool-112-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 176338 DEBUG (simCloudManagerPool-112-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
   [junit4]   2> 176339 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 176374 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10003_solr
   [junit4]   2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10003_solr
   [junit4]   2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 176374 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 176374 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 176374 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 176374 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [5808042313008762], now=5808042332719662
   [junit4]   2> 176375 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"14a2623ae8467aTb6dcrwj17bt0ghdhsu6uyamuz",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":5808042313008762,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808042313008762],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 176376 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_restore_trigger]
   [junit4]   2> 176446 DEBUG (ScheduledTrigger-113-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10003_solr
   [junit4]   2> 176447 DEBUG (AutoscalingActionExecutor-114-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14a2623ae8467aTb6dcrwj17bt0ghdhsu6uyamuz",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":5808042313008762,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808042313008762],
   [junit4]   2>     "_enqueue_time_":5808042440598362,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 176447 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
   [junit4]   2> 176447 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 176447 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 176447 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 176447 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 176447 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		3
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	3
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		0
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        3
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             1
   [junit4]   2> 176448 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 176449 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_restore_trigger
   [junit4]   2> 176449 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testListeners
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testEventFromRestoredState
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 176539 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue
   [junit4]   2> 176539 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
   [junit4]   2> 176556 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 176556 DEBUG (AutoscalingActionExecutor-114-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 176556 DEBUG (AutoscalingActionExecutor-114-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_restore_trigger after 100ms
   [junit4]   2> 176556 DEBUG (AutoscalingActionExecutor-114-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 5470 ms for event id=14a2623ae8467aTb6dcrwj17bt0ghdhsu6uyamuz
   [junit4]   2> 176609 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 176610 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 176610 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 176610 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 176610 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 176610 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 176610 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 176611 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 176611 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 176611 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 176612 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 176667 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 176667 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
   [junit4]   2> 176668 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion -1
   [junit4]   2> 176668 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 176718 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 176755 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 176755 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 176755 DEBUG (ScheduledTrigger-116-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 176882 DEBUG (simCloudManagerPool-115-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 176883 DEBUG (simCloudManagerPool-115-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
   [junit4]   2> 176942 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
   [junit4]   2> 176942 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 176942 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
   [junit4]   2> 176964 INFO  (simCloudManagerPool-115-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 176964 DEBUG (simCloudManagerPool-115-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 176965 DEBUG (simCloudManagerPool-115-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 176965 DEBUG (simCloudManagerPool-115-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
   [junit4]   2> 177066 INFO  (simCloudManagerPool-115-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 177066 DEBUG (simCloudManagerPool-115-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 177066 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
   [junit4]   2> 177067 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 177067 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true}
   [junit4]   2> 177067 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
   [junit4]   2> 177080 DEBUG (ScheduledTrigger-116-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 177080 DEBUG (ScheduledTrigger-116-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 177081 DEBUG (ScheduledTrigger-116-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 177081 DEBUG (ScheduledTrigger-116-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 5808077654319262
   [junit4]   2> 177101 DEBUG (ScheduledTrigger-116-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 177153 DEBUG (ScheduledTrigger-116-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10004_solr] added at times [5808077654319262], now=5808081303618062
   [junit4]   2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5808077654319262,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808077654319262],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1]
   [junit4]   2> 177210 DEBUG (ScheduledTrigger-116-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10004_solr
   [junit4]   2> 177211 DEBUG (AutoscalingActionExecutor-117-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5808077654319262,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808077654319262],
   [junit4]   2>     "_enqueue_time_":5808081326400762,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 177211 INFO  (AutoscalingActionExecutor-117-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5808077654319262,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808077654319262],
   [junit4]   2>     "_enqueue_time_":5808081326400762,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 177226 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 177226 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 177226 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 177226 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 177226 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 177227 WARN  (AutoscalingActionExecutor-117-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed.
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:546) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:319) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_152]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 177227 DEBUG (AutoscalingActionExecutor-117-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 798 ms for event id=14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2
   [junit4]   2> 177227 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 177227 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 177227 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 177229 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
   [junit4]   2> 177283 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 177283 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 177283 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion -1
   [junit4]   2> 177283 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
   [junit4]   2> 177283 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 177284 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10000_solr
   [junit4]   2> 177335 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 177335 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true}
   [junit4]   2> 177355 DEBUG (ScheduledTrigger-119-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 177356 DEBUG (ScheduledTrigger-119-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr
   [junit4]   2> 177371 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
   [junit4]   2> 177372 DEBUG (ScheduledTrigger-119-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5808077654319262,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808077654319262],
   [junit4]   2>     "_dequeue_time_":5808092202008612,
   [junit4]   2>     "_enqueue_time_":5808081326400762,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 177373 DEBUG (ScheduledTrigger-119-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1]
   [junit4]   2> 177445 DEBUG (ScheduledTrigger-119-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 177445 DEBUG (ScheduledTrigger-119-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 177495 DEBUG (AutoscalingActionExecutor-120-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5808077654319262,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808077654319262],
   [junit4]   2>     "_dequeue_time_":5808092202008612,
   [junit4]   2>     "_enqueue_time_":5808081326400762,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 177496 INFO  (AutoscalingActionExecutor-120-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5808077654319262,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808077654319262],
   [junit4]   2>     "_dequeue_time_":5808092202008612,
   [junit4]   2>     "_enqueue_time_":5808081326400762,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 177499 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		2
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	2
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		1
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		127.0.0.1:10000_solr
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        2
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             2
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1
   [junit4]   2> 177500 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         2
   [junit4]   2> 177526 DEBUG (AutoscalingActionExecutor-120-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 177542 DEBUG (AutoscalingActionExecutor-120-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 177542 DEBUG (AutoscalingActionExecutor-120-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 2335 ms for event id=14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2
   [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testNodeMarkersRegistration
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testNodeLostTriggerRestoreState
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 177656 DEBUG (ScheduledTrigger-119-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 177884 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling
   [junit4]   2> 177884 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14
   [junit4]   2> 177884 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10000_solr
   [junit4]   2> 177884 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 177884 DEBUG (ScheduledTrigger-119-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 177884 DEBUG (ScheduledTrigger-119-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 177885 DEBUG (ScheduledTrigger-119-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 177885 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 177885 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 177886 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 177891 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 177931 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 177932 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 177932 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 177987 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 177987 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15
   [junit4]   2> 178080 DEBUG (simCloudManagerPool-121-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178094 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion -1
   [junit4]   2> 178094 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
   [junit4]   2> 178094 DEBUG (simCloudManagerPool-121-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16
   [junit4]   2> 178117 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 178130 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178130 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 15
   [junit4]   2> 178130 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16
   [junit4]   2> 178132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 178132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
   [junit4]   2> 178132 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178133 DEBUG (simCloudManagerPool-121-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178133 DEBUG (simCloudManagerPool-121-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17
   [junit4]   2> 178135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17
   [junit4]   2> 178135 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 178135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 178136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true}
   [junit4]   2> 178136 DEBUG (ScheduledTrigger-122-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 178136 DEBUG (ScheduledTrigger-122-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17
   [junit4]   2> 178152 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178152 DEBUG (simCloudManagerPool-121-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178153 DEBUG (simCloudManagerPool-121-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 18
   [junit4]   2> 178188 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 178188 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178188 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178209 DEBUG (ScheduledTrigger-122-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 178209 DEBUG (ScheduledTrigger-122-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178210 DEBUG (ScheduledTrigger-122-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178210 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18
   [junit4]   2> 178211 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 178211 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 178211 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true}
   [junit4]   2> 178211 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18
   [junit4]   2> 178211 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 178211 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178236 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 178236 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 178236 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 5808135434972662
   [junit4]   2> 178236 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10005_solr] added at times [5808135434972662], now=5808135437518362
   [junit4]   2> 178237 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"14a277e968adf6Tb6dcrwj17bt0ghdhsu6uyamv6",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5808135434972662,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808135434972662],
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 178237 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1]
   [junit4]   2> 178238 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10005_solr
   [junit4]   2> 178238 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14a277e968adf6Tb6dcrwj17bt0ghdhsu6uyamv6",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5808135434972662,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808135434972662],
   [junit4]   2>     "_enqueue_time_":5808135484706712,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 178238 INFO  (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger1
   [junit4]   2> 178254 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 178254 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 178254 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 178254 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 783 ms for event id=14a277e968adf6Tb6dcrwj17bt0ghdhsu6uyamv6
   [junit4]   2> 178362 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 178362 DEBUG (ScheduledTrigger-122-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 178362 DEBUG (ScheduledTrigger-122-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 178384 DEBUG (ScheduledTrigger-122-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 178384 DEBUG (ScheduledTrigger-122-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 178384 DEBUG (ScheduledTrigger-122-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 178398 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 178398 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 178399 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 5808143544140212
   [junit4]   2> 178399 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:10005_solr] added at times [5808143544140212], now=5808143549185512
   [junit4]   2> 178399 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"14a279ccc0c1b4Tb6dcrwj17bt0ghdhsu6uyamv9",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":5808143544140212,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808143544140212],
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 178399 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1]
   [junit4]   2> 178400 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10005_solr
   [junit4]   2> 178401 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14a279ccc0c1b4Tb6dcrwj17bt0ghdhsu6uyamv9",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":5808143544140212,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5808143544140212],
   [junit4]   2>     "_enqueue_time_":5808143567743712,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 178401 INFO  (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration last action at 5808135539962012 time = 5808143655653162
   [junit4]   2> 178401 INFO  (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger2
   [junit4]   2> 178419 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 178419 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 178419 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 178419 DEBUG (AutoscalingActionExecutor-123-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 913 ms for event id=14a279ccc0c1b4Tb6dcrwj17bt0ghdhsu6uyamv9
   [junit4]   2> 178420 DEBUG (simCloudManagerPool-121-thread-8) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178420 DEBUG (simCloudManagerPool-121-thread-8) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 19
   [junit4]   2> 178421 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 19
   [junit4]   2> 178421 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 178421 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger1 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 178421 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 19, lastZnodeVersion 19
   [junit4]   2> 178422 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
   [junit4]   2> 178455 DEBUG (ScheduledTrigger-122-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
   [junit4]   2> 178456 DEBUG (simCloudManagerPool-121-thread-9) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 178456 DEBUG (simCloudManagerPool-121-thread-9) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 20
   [junit4]   2> 178457 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 20
   [junit4]   2> 178457 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 178457 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger2 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 178458 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion 20
   [junit4]   2> 178458 DEBUG (ScheduledTrigger-122-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 3
   [junit4]   2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr
   [junit4]   2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178564 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 178565 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 178565 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 178565 DEBUG (ScheduledTrigger-122-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4] 

[...truncated too long message...]

987-thread-3) [n:127.0.0.1:42668_solr c:hdfsbackuprestore s:shard2 r:core_node12 x:hdfsbackuprestore_shard2_replica_p10] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore.shard2.leader, tag=3f9292a8
   [junit4]   2> 6718968 INFO  (coreCloseExecutor-13986-thread-1) [n:127.0.0.1:40759_solr c:hdfsbackuprestore s:shard2 r:core_node11 x:hdfsbackuprestore_shard2_replica_t8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore.shard2.leader, tag=a7c2bfd
   [junit4]   2> 6718999 INFO  (jetty-closer-57564-thread-2) [    ] o.a.s.c.Overseer Overseer (id=72438645494841350-127.0.0.1:40759_solr-n_0000000000) closing
   [junit4]   2> 6718999 INFO  (OverseerStateUpdate-72438645494841350-127.0.0.1:40759_solr-n_0000000000) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:40759_solr
   [junit4]   2> 6719000 WARN  (OverseerAutoScalingTriggerThread-72438645494841350-127.0.0.1:40759_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 6719003 INFO  (jetty-closer-57564-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@381f9003{/solr,null,UNAVAILABLE}
   [junit4]   2> 6719003 INFO  (jetty-closer-57564-thread-1) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 6719004 INFO  (jetty-closer-57564-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6298e1cc{/solr,null,UNAVAILABLE}
   [junit4]   2> 6719005 INFO  (jetty-closer-57564-thread-2) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 6719006 ERROR (SUITE-TestHdfsCloudBackupRestore-seed#[93925E63CC51A4B4]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 6719014 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[93925E63CC51A4B4]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45419 45419
   [junit4]   2> 6719074 INFO  (Thread-10622) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45419 45419
   [junit4]   2> 6719075 WARN  (Thread-10622) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/collections/hdfsbackuprestore/terms/shard2
   [junit4]   2> 	5	/solr/collections/hdfsbackuprestore/terms/shard1
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	3	/solr/collections/hdfsbackuprestore_restored/terms/shard1
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/configs/conf1
   [junit4]   2> 	2	/solr/collections/hdfsbackuprestore_restored/terms/shard2
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	16	/solr/collections/hdfsbackuprestore/state.json
   [junit4]   2> 	15	/solr/collections/hdfsbackuprestore_restored/state.json
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/live_nodes
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_93925E63CC51A4B4-001
   [junit4]   2> Apr 11, 2018 8:03:15 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 39 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {shard_s=PostingsFormat(name=LuceneFixedGap), id=FST50}, docValues:{}, maxPointsInLeafNode=1843, maxMBSortInHeap=7.6243392989430845, sim=RandomSimilarity(queryNorm=false): {}, locale=uk-UA, timezone=Africa/Asmera
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=149539104,total=479723520
   [junit4]   2> NOTE: All tests run in this JVM: [TestSmileRequest, SchemaVersionSpecificBehaviorTest, TestReload, TestNumericTokenStream, ZkShardTermsTest, RegexBytesRefFilterTest, TestDistributedGrouping, ResourceLoaderTest, TestRTGBase, TestScoreJoinQPScore, TestAuthenticationFramework, TestSweetSpotSimilarityFactory, ReplicaListTransformerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestFuzzyAnalyzedSuggestions, FastVectorHighlighterTest, TestOnReconnectListenerSupport, XsltUpdateRequestHandlerTest, SyncSliceTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestReversedWildcardFilterFactory, TestUseDocValuesAsStored, TestSkipOverseerOperations, TestFoldingMultitermQuery, TestLuceneIndexBackCompat, DistributedQueueTest, TestMaxScoreQueryParser, TestCustomStream, TestBackupRepositoryFactory, TestJsonFacetsWithNestedObjects, SuggestComponentContextFilterQueryTest, TestPartialUpdateDeduplication, PeerSyncTest, TestIntervalFaceting, TestZkAclsWithHadoopAuth, TestCursorMarkWithoutUniqueKey, CurrencyFieldTypeTest, BasicAuthStandaloneTest, SuggesterTest, OverseerModifyCollectionTest, TestNumericTerms64, TestCloudPseudoReturnFields, WrapperMergePolicyFactoryTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestReloadDeadlock, TestDelegationWithHadoopAuth, TestSchemaNameResource, TestSubQueryTransformerDistrib, HdfsUnloadDistributedZkTest, JsonLoaderTest, SignatureUpdateProcessorFactoryTest, TestSimpleTextCodec, TestRTimerTree, TestExportWriter, TestFieldCacheSort, TestSchemaManager, CollectionTooManyReplicasTest, TestComplexPhraseLeadingWildcard, SOLR749Test, ZkControllerTest, LeaderFailoverAfterPartitionTest, TestSolrJ, SortSpecParsingTest, TestDistribDocBasedVersion, NodeAddedTriggerTest, BufferStoreTest, TestPayloadScoreQParserPlugin, LargeFieldTest, TestReplicationHandler, TestDefaultStatsCache, TestPKIAuthenticationPlugin, TestCloudNestedDocsSort, DirectSolrConnectionTest, CloudExitableDirectoryReaderTest, MetricsConfigTest, AnalyticsQueryTest, TestBM25SimilarityFactory, TestSearchPerf, TestExpandComponent, EnumFieldTest, TriggerCooldownIntegrationTest, TestSimDistributedQueue, TestMultiValuedNumericRangeQuery, TestNRTOpen, SecurityConfHandlerTest, SuggesterFSTTest, HdfsChaosMonkeyNothingIsSafeTest, TestHighFrequencyDictionaryFactory, ExitableDirectoryReaderTest, MetricTriggerIntegrationTest, ParsingFieldUpdateProcessorsTest, TestBlendedInfixSuggestions, TestRecovery, LegacyCloudClusterPropTest, TestLeaderInitiatedRecoveryThread, TestCoreAdminApis, IndexSchemaTest, HighlighterTest, SchemaApiFailureTest, TestRestManager, TestQueryUtils, TestFieldCacheWithThreads, TestConfigSetsAPIZkFailure, SolrCLIZkUtilsTest, TestSortByMinMaxFunction, TestTolerantUpdateProcessorRandomCloud, TestInPlaceUpdatesStandalone, DocumentBuilderTest, DistributedFacetPivotWhiteBoxTest, HdfsRecoverLeaseTest, DistributedFacetSimpleRefinementLongTailTest, TestSolrConfigHandlerConcurrent, SearchRateTriggerTest, TestClusterProperties, MultiSolrCloudTestCaseTest, TestRawTransformer, ShardSplitTest, ActionThrottleTest, TermsComponentTest, TestDocTermOrds, ConnectionManagerTest, DistributedIntervalFacetingTest, TestSolrFieldCacheBean, CoreAdminOperationTest, CloneFieldUpdateProcessorFactoryTest, V2StandaloneTest, TestExecutePlanAction, QueryElevationComponentTest, SortByFunctionTest, TestSolrCloudSnapshots, SliceStateTest, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, SpellCheckComponentTest, TestGroupingSearch, SolrCmdDistributorTest, TestSort, TestFiltering, TestFunctionQuery, BasicFunctionalityTest, TestBadConfig, SoftAutoCommitTest, CSVRequestHandlerTest, TestLFUCache, TestCrossCoreJoin, TestDistributedMissingSort, TokenizerChainTest, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerSchemaAPI, ConnectionReuseTest, AssignBackwardCompatibilityTest, CleanupOldIndexTest, ConfigSetsAPITest, CreateCollectionCleanupTest, CreateRoutedAliasTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DocValuesNotIndexedTest, ForceLeaderTest, LIRRollingUpdatesTest, OverseerTaskQueueTest, RemoteQueryErrorTest, ReplaceNodeNoTargetTest, ReplaceNodeTest, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestPullReplicaErrorHandling, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestShortCircuitedRequests, TestSolrCloudWithSecureImpersonation, TestStressLiveNodes, TriLevelCompositeIdRoutingTest, CollectionReloadTest, ConcurrentDeleteAndCreateCollectionTest, CustomCollectionTest, SimpleCollectionCreateDeleteTest, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestHdfsCloudBackupRestore]
   [junit4] Completed [604/800 (3!)] on J1 in 25.97s, 1 test, 1 error <<< FAILURES!

[...truncated 657 lines...]
   [junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J1-20180411_181116_5877357961297125018620.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid6943.hprof ...
   [junit4] Heap dump file created [475496812 bytes in 3.465 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 9284 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid6943.hprof

Total time: 495 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 197 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/197/

1 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
	at __randomizedtesting.SeedInfo.seed([1D1E264F0B72E37F:9523AF3031B202D2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:379)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13044 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_1D1E264F0B72E37F-001/init-core-data-001
   [junit4]   2> 825628 INFO  (SUITE-TestTriggerIntegration-seed#[1D1E264F0B72E37F]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 825630 INFO  (SUITE-TestTriggerIntegration-seed#[1D1E264F0B72E37F]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 825630 INFO  (SUITE-TestTriggerIntegration-seed#[1D1E264F0B72E37F]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 825791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 825791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 825792 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 825792 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4] IGNOR/A 0.06s J1 | TestTriggerIntegration.testCooldown
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J1 | TestTriggerIntegration.testEventFromRestoredState
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 825838 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 825840 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 825840 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 826117 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
   [junit4]   2> 826117 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 826117 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 826152 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 826152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 826153 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 826153 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) ~[?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) ~[?:1.8.0_152]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:169) [java/:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 826232 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 826232 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 826232 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 826232 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 826252 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 826252 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 826277 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 826277 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 826278 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 826278 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 826314 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 826314 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 826314 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 826314 DEBUG (ScheduledTrigger-634-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 826335 DEBUG (ScheduledTrigger-634-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 826716 DEBUG (ScheduledTrigger-634-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 826862 DEBUG (ScheduledTrigger-634-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 826883 DEBUG (ScheduledTrigger-634-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 826883 DEBUG (simCloudManagerPool-633-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 826884 DEBUG (simCloudManagerPool-633-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 826956 DEBUG (ScheduledTrigger-634-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 826956 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 826956 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 826957 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 826992 DEBUG (ScheduledTrigger-634-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 827005 DEBUG (simCloudManagerPool-633-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 827005 DEBUG (simCloudManagerPool-633-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 827008 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 827008 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 827008 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 827008 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 827009 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10002_solr
   [junit4]   2> 827009 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10002_solr
   [junit4]   2> 827009 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 827009 DEBUG (ScheduledTrigger-634-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 827009 DEBUG (ScheduledTrigger-634-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 827028 DEBUG (ScheduledTrigger-634-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 827067 DEBUG (simCloudManagerPool-633-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 827067 DEBUG (simCloudManagerPool-633-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 827068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 827068 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 827068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 827068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 827068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 827068 DEBUG (ScheduledTrigger-634-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 827068 DEBUG (ScheduledTrigger-634-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 827068 WARN  (ScheduledTrigger-634-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger ran but was already closed
   [junit4]   2> 827068 DEBUG (ScheduledTrigger-634-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10002_solr] added at times [5681710672540093], now=5681713628869643
   [junit4]   2> 827068 ERROR (ScheduledTrigger-634-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Unexpected exception in NodeAddedTrigger
   [junit4]   2> java.lang.RuntimeException: Trigger has been closed
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.NodeAddedTrigger.run(NodeAddedTrigger.java:121) [java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:589) [java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 827070 DEBUG (ScheduledTrigger-634-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"142f7c5a5a71bdT628nl585kreui5phdtg4uubfu",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":5681710672540093,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681710672540093],
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 827255 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
   [junit4]   2> 827255 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 827255 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 827255 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 827255 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 827255 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		3
   [junit4]   2> 827255 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	3
   [junit4]   2> 827255 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		0
   [junit4]   2> 827272 DEBUG (ScheduledTrigger-634-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_restore_trigger]
   [junit4]   2> 827276 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 827276 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 827276 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 827276 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 827289 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 827289 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        3
   [junit4]   2> 827289 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             1
   [junit4]   2> 827289 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 827290 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_restore_trigger
   [junit4]   2> 827290 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeAddedTriggerRestoreState -Dtests.seed=1D1E264F0B72E37F -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=Europe/Sofia -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 1.50s J1 | TestTriggerIntegration.testNodeAddedTriggerRestoreState <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The trigger did not fire at all
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1D1E264F0B72E37F:9523AF3031B202D2]:0)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:379)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 827317 DEBUG (AutoscalingActionExecutor-635-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"142f7c5a5a71bdT628nl585kreui5phdtg4uubfu",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":5681710672540093,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681710672540093],
   [junit4]   2>     "_enqueue_time_":5681723855762143,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 827353 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling
   [junit4]   2> 827353 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 827359 WARN  (AutoscalingActionExecutor-635-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> java.lang.NullPointerException: null
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:320) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_152]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 827366 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 827353 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 827374 DEBUG (ScheduledTrigger-634-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10002_solr
   [junit4]   2> 827394 DEBUG (AutoscalingActionExecutor-635-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 827395 DEBUG (AutoscalingActionExecutor-635-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_restore_trigger after 100ms
   [junit4]   2> 827395 DEBUG (AutoscalingActionExecutor-635-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 3872 ms for event id=142f7c5a5a71bdT628nl585kreui5phdtg4uubfu
   [junit4]   2> 827448 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 827448 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 827448 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 827448 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 827448 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 827448 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) ~[?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) ~[?:1.8.0_152]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:169) [java/:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 827486 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 827486 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 827486 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 827501 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 827501 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 827501 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1
   [junit4]   2> 827501 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 827557 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 827557 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 827557 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 827557 DEBUG (ScheduledTrigger-637-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 827610 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 827611 DEBUG (simCloudManagerPool-636-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 827612 DEBUG (simCloudManagerPool-636-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
   [junit4]   2> 827612 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 827613 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 827613 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 827615 DEBUG (simCloudManagerPool-636-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 827615 DEBUG (simCloudManagerPool-636-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
   [junit4]   2> 827615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 827615 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 827616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 827616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true}
   [junit4]   2> 827616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 827616 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 827616 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 827617 DEBUG (simCloudManagerPool-636-thread-3) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 827617 DEBUG (simCloudManagerPool-636-thread-3) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
   [junit4]   2> 827638 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 827638 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 827638 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 827639 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 827639 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 827639 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 827639 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true}
   [junit4]   2> 827640 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 827646 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 827646 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 827646 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 5681742528646893
   [junit4]   2> 827646 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [5681742528646893], now=5681742530669693
   [junit4]   2> 827646 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"142f83c5200eedT628nl585kreui5phdtg4uubfw",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":5681742528646893,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681742528646893],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 827647 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1]
   [junit4]   2> 827654 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10003_solr
   [junit4]   2> 827654 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"142f83c5200eedT628nl585kreui5phdtg4uubfw",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":5681742528646893,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681742528646893],
   [junit4]   2>     "_enqueue_time_":5681742593939193,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 827655 INFO  (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger2
   [junit4]   2> 827655 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 827655 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 827655 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 827655 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 52 ms for event id=142f83c5200eedT628nl585kreui5phdtg4uubfw
   [junit4]   2> 827772 DEBUG (ScheduledTrigger-637-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 827776 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 827776 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 827776 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 5681749017754493
   [junit4]   2> 827776 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [5681749017754493], now=5681749019890893
   [junit4]   2> 827776 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"142f8547e7fb7dT628nl585kreui5phdtg4uubfz",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5681749017754493,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681749017754493],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 827776 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1]
   [junit4]   2> 827793 DEBUG (ScheduledTrigger-637-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 827793 DEBUG (ScheduledTrigger-637-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 827794 DEBUG (ScheduledTrigger-637-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10003_solr
   [junit4]   2> 827794 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"142f8547e7fb7dT628nl585kreui5phdtg4uubfz",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5681749017754493,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681749017754493],
   [junit4]   2>     "_enqueue_time_":5681749035248293,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 827794 INFO  (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration last action at 5681742952526093 time = 5681749936144493
   [junit4]   2> 827794 INFO  (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger1
   [junit4]   2> 827808 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 827808 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 827809 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 827809 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 720 ms for event id=142f8547e7fb7dT628nl585kreui5phdtg4uubfz
   [junit4]   2> 827810 DEBUG (simCloudManagerPool-636-thread-8) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 827810 DEBUG (simCloudManagerPool-636-thread-8) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
   [junit4]   2> 827811 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 827812 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 827812 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger1 - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 827812 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 827812 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
   [junit4]   2> 827846 DEBUG (simCloudManagerPool-636-thread-9) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 827846 DEBUG (simCloudManagerPool-636-thread-9) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
   [junit4]   2> 827846 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
   [junit4]   2> 827847 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 827847 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger2 - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 827847 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
   [junit4]   2> 827849 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 2
   [junit4]   2> 827849 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10003_solr
   [junit4]   2> 827849 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10003_solr]
   [junit4]   2> 827849 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"142f8621775091T628nl585kreui5phdtg4uubg2",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":5681752667803793,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681752667803793],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 827849 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger2, .auto_add_replicas, node_added_trigger2, node_lost_trigger1, node_added_trigger1]
   [junit4]   2> 827850 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"142f8621775091T628nl585kreui5phdtg4uubg2",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":5681752667803793,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681752667803793],
   [junit4]   2>     "_enqueue_time_":5681752690337493,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 827850 INFO  (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_lost_trigger1
   [junit4]   2> 827851 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger2 after 100ms
   [junit4]   2> 827851 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 827851 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 827851 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms
   [junit4]   2> 827851 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 827851 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 31 ms for event id=142f8621775091T628nl585kreui5phdtg4uubg2
   [junit4]   2> 827959 DEBUG (ScheduledTrigger-637-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 2
   [junit4]   2> 827959 DEBUG (ScheduledTrigger-637-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10003_solr
   [junit4]   2> 827959 DEBUG (ScheduledTrigger-637-thread-2) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10003_solr]
   [junit4]   2> 827959 DEBUG (ScheduledTrigger-637-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"142f876873e959T628nl585kreui5phdtg4uubg5",
   [junit4]   2>   "source":"node_lost_trigger2",
   [junit4]   2>   "eventTime":5681758153730393,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681758153730393],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 827959 DEBUG (ScheduledTrigger-637-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger2, .auto_add_replicas, node_added_trigger2, node_lost_trigger1, node_added_trigger1]
   [junit4]   2> 827976 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"142f876873e959T628nl585kreui5phdtg4uubg5",
   [junit4]   2>   "source":"node_lost_trigger2",
   [junit4]   2>   "eventTime":5681758153730393,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681758153730393],
   [junit4]   2>     "_enqueue_time_":5681758174760143,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 827976 INFO  (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration last action at 5681752742305943 time = 5681759015496293
   [junit4]   2> 827976 INFO  (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_lost_trigger2
   [junit4]   2> 827976 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testTriggerThrottling
   [junit4]   2> 827976 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 827976 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 827976 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 827976 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		2
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	2
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		127.0.0.1:10003_solr
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        5
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             7
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- SUCCEEDED       1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger2
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- SUCCEEDED       1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- SUCCEEDED       1
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger2
   [junit4]   2> 827977 INFO  (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4] IGNOR/A 0.00s J1 | TestTriggerIntegration.testNodeMarkersRegistration
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J1 | TestTriggerIntegration.testListeners
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J1 | TestTriggerIntegration.testSearchRate
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J1 | TestTriggerIntegration.testNodeLostTrigger
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4] IGNOR/A 0.00s J1 | TestTriggerIntegration.testNodeLostTriggerRestoreState
   [junit4]    > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 828048 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger2 after 100ms
   [junit4]   2> 828050 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 828050 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 828050 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms
   [junit4]   2> 828050 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 828050 DEBUG (AutoscalingActionExecutor-638-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 3694 ms for event id=142f876873e959T628nl585kreui5phdtg4uubg5
   [junit4]   2> 828211 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828212 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10003_solr
   [junit4]   2> 828212 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 828212 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 828212 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 2
   [junit4]   2> 828213 DEBUG (ScheduledTrigger-637-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 2
   [junit4]   2> 828213 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 828213 DEBUG (ScheduledTrigger-637-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 828213 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
   [junit4]   2> 828213 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
   [junit4]   2> 828214 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10003_solr
   [junit4]   2> 828214 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 828214 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 828214 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 828214 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 828214 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 828232 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 828232 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 828232 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 828261 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 828261 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14
   [junit4]   2> 828276 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1
   [junit4]   2> 828276 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
   [junit4]   2> 828276 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 828277 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 828277 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 828277 DEBUG (ScheduledTrigger-640-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828304 DEBUG (ScheduledTrigger-640-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828341 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828377 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828377 DEBUG (simCloudManagerPool-639-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 828378 DEBUG (simCloudManagerPool-639-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15
   [junit4]   2> 828378 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
   [junit4]   2> 828379 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 828379 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
   [junit4]   2> 828403 DEBUG (simCloudManagerPool-639-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 828403 DEBUG (simCloudManagerPool-639-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16
   [junit4]   2> 828403 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16
   [junit4]   2> 828404 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 828404 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 828404 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 828404 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
   [junit4]   2> 828414 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 828414 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 828414 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 828414 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 5681780930016993
   [junit4]   2> 828466 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 828466 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 828466 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 828466 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10004_solr] added at times [5681780930016993], now=5681783533726043
   [junit4]   2> 828466 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"142f8cb60676e1T628nl585kreui5phdtg4uubg8",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":5681780930016993,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681780930016993],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 828470 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger]
   [junit4]   2> 828523 DEBUG (ScheduledTrigger-640-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10004_solr
   [junit4]   2> 828523 DEBUG (AutoscalingActionExecutor-641-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"142f8cb60676e1T628nl585kreui5phdtg4uubg8",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":5681780930016993,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681780930016993],
   [junit4]   2>     "_enqueue_time_":5681783741718243,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 828524 DEBUG (AutoscalingActionExecutor-641-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 828524 DEBUG (AutoscalingActionExecutor-641-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 828524 DEBUG (AutoscalingActionExecutor-641-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 33 ms for event id=142f8cb60676e1T628nl585kreui5phdtg4uubg8
   [junit4]   2> 828548 DEBUG (simCloudManagerPool-639-thread-5) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 828549 DEBUG (simCloudManagerPool-639-thread-5) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17
   [junit4]   2> 828564 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17
   [junit4]   2> 828564 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17
   [junit4]   2> 828633 DEBUG (ScheduledTrigger-640-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 828633 DEBUG (ScheduledTrigger-640-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 828633 DEBUG (ScheduledTrigger-640-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		3
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	3
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		0
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        3
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             2
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 828637 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger
   [junit4]   2> 828638 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4]   2> 828638 INFO  (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- SUCCEEDED       1
   [junit4]   2> 828656 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue
   [junit4]   2> 828656 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 18
   [junit4]   2> 828656 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18
   [junit4]   2> 828656 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 828656 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 828656 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18
   [junit4]   2> 828656 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 828656 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) ~[?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048) ~[?:1.8.0_152]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:181) [java/:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 828657 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 828657 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 828657 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 828685 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 828686 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 828686 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 828722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   2> 828723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 19
   [junit4]   2> 828723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 19, lastZnodeVersion -1
   [junit4]   2> 828723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 19
   [junit4]   2> 828743 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10004_solr]
   [junit4]   2> 828743 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 828743 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 19, lastZnodeVersion 19
   [junit4]   2> 828759 DEBUG (ScheduledTrigger-643-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828797 DEBUG (ScheduledTrigger-643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828832 DEBUG (ScheduledTrigger-643-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828833 DEBUG (simCloudManagerPool-642-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 828833 DEBUG (simCloudManagerPool-642-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 20
   [junit4]   2> 828834 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 20
   [junit4]   2> 828835 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 828835 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion 20
   [junit4]   2> 828849 INFO  (simCloudManagerPool-642-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 828849 DEBUG (simCloudManagerPool-642-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 828849 DEBUG (simCloudManagerPool-642-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 828849 DEBUG (simCloudManagerPool-642-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 21
   [junit4]   2> 828850 INFO  (simCloudManagerPool-642-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 828850 DEBUG (simCloudManagerPool-642-thread-2) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 828850 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 21
   [junit4]   2> 828850 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10004_solr]
   [junit4]   2> 828850 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true}
   [junit4]   2> 828850 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 21, lastZnodeVersion 21
   [junit4]   2> 828852 DEBUG (ScheduledTrigger-643-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 828868 DEBUG (ScheduledTrigger-643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828885 DEBUG (ScheduledTrigger-643-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 828885 DEBUG (ScheduledTrigger-643-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 5681804489279293
   [junit4]   2> 828905 DEBUG (ScheduledTrigger-643-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 828906 DEBUG (ScheduledTrigger-643-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 828906 DEBUG (ScheduledTrigger-643-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 828906 DEBUG (ScheduledTrigger-643-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10005_solr] added at times [5681804489279293], now=5681805542652543
   [junit4]   2> 828907 DEBUG (ScheduledTrigger-643-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"142f923244473dT628nl585kreui5phdtg4uubgb",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5681804489279293,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681804489279293],
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 828908 DEBUG (ScheduledTrigger-643-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1]
   [junit4]   2> 828958 DEBUG (ScheduledTrigger-643-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10005_solr
   [junit4]   2> 828958 DEBUG (AutoscalingActionExecutor-644-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"142f923244473dT628nl585kreui5phdtg4uubgb",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5681804489279293,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681804489279293],
   [junit4]   2>     "_enqueue_time_":5681805607977843,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 828958 INFO  (AutoscalingActionExecutor-644-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"142f923244473dT628nl585kreui5phdtg4uubgb",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5681804489279293,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681804489279293],
   [junit4]   2>     "_enqueue_time_":5681805607977843,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 828958 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 828959 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
   [junit4]   2> 828959 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
   [junit4]   2> 828959 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
   [junit4]   2> 828959 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 828959 WARN  (AutoscalingActionExecutor-644-thread-1) [    ] o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed.
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:546) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:319) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_152]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 828960 DEBUG (AutoscalingActionExecutor-644-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 79 ms for event id=142f923244473dT628nl585kreui5phdtg4uubgb
   [junit4]   2> 828960 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
   [junit4]   2> 828960 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
   [junit4]   2> 828960 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 828978 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 21
   [junit4]   2> 828979 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 828979 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 828979 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 21, lastZnodeVersion -1
   [junit4]   2> 828979 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 21
   [junit4]   2> 828980 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 828980 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10001_solr
   [junit4]   2> 828994 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 828994 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true}
   [junit4]   2> 828995 DEBUG (ScheduledTrigger-646-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 828995 DEBUG (ScheduledTrigger-646-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
   [junit4]   2> 828995 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 21, lastZnodeVersion 21
   [junit4]   2> 828995 DEBUG (ScheduledTrigger-646-thread-2) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"142f923244473dT628nl585kreui5phdtg4uubgb",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5681804489279293,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681804489279293],
   [junit4]   2>     "_dequeue_time_":5681809979889843,
   [junit4]   2>     "_enqueue_time_":5681805607977843,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 829017 DEBUG (ScheduledTrigger-646-thread-2) [    ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1]
   [junit4]   2> 829051 DEBUG (ScheduledTrigger-646-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 829051 DEBUG (ScheduledTrigger-646-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 829052 DEBUG (AutoscalingActionExecutor-647-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"142f923244473dT628nl585kreui5phdtg4uubgb",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5681804489279293,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681804489279293],
   [junit4]   2>     "_dequeue_time_":5681809979889843,
   [junit4]   2>     "_enqueue_time_":5681805607977843,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 829052 INFO  (AutoscalingActionExecutor-647-thread-1) [    ] o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"142f923244473dT628nl585kreui5phdtg4uubgb",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5681804489279293,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5681804489279293],
   [junit4]   2>     "_dequeue_time_":5681809979889843,
   [junit4]   2>     "_enqueue_time_":5681805607977843,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		2
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	2
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		1
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		127.0.0.1:10001_solr
   [junit4]   2> 829080 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        2
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             2
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1
   [junit4]   2> 829081 INFO  (TEST-TestTriggerIntegration.testEventQueue-seed#[1D1E264F0B72E37F]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         2
   [junit4]   2> 829081 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [

[...truncated too long message...]

y-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1017095066
     [copy] Copying 40 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1017095066
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1017095066

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1390607118
     [copy] Copying 246 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1390607118
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1390607118

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 27ms :: artifacts dl 1ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 502 minutes 24 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

Re: [JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 196 - Failure

Posted by Simon Willnauer <si...@gmail.com>.
this is an endless merge loop due to merge policy accounting issues
that were not implemented yet. I opened
https://issues.apache.org/jira/browse/LUCENE-8246

On Mon, Apr 9, 2018 at 12:19 PM, Simon Willnauer
<si...@gmail.com> wrote:
> I am looking into this
>
> On Mon, Apr 9, 2018 at 12:06 PM, Apache Jenkins Server
> <je...@builds.apache.org> wrote:
>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/196/
>>
>> 5 tests failed.
>> FAILED:  org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention
>>
>> Error Message:
>> GC overhead limit exceeded
>>
>> Stack Trace:
>> java.lang.OutOfMemoryError: GC overhead limit exceeded
>>         at __randomizedtesting.SeedInfo.seed([248C95A59BE17434:3432A38A78701E91]:0)
>>         at java.util.HashMap$KeySet.iterator(HashMap.java:917)
>>         at java.util.HashSet.iterator(HashSet.java:173)
>>         at org.apache.lucene.index.IndexWriter.maxNumSegmentsMergesPending(IndexWriter.java:2216)
>>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2190)
>>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2119)
>>         at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention(TestSoftDeletesRetentionMergePolicy.java:266)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.lang.reflect.Method.invoke(Method.java:498)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>
>>
>> FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy
>>
>> Error Message:
>> 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:     1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]         at java.lang.Thread.yield(Native Method)         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>>
>> Stack Trace:
>> com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:
>>    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>>         at java.lang.Thread.yield(Native Method)
>>         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
>>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
>>         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
>>         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
>>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
>>         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
>>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
>>         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
>>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
>>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
>>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
>>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>>         at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
>>
>>
>> FAILED:  org.apache.lucene.search.TestCustomSearcherSort.testFieldSortCustomSearcher
>>
>> Error Message:
>> GC overhead limit exceeded
>>
>> Stack Trace:
>> java.lang.OutOfMemoryError: GC overhead limit exceeded
>>         at org.apache.lucene.util.packed.PackedLongValues$Iterator.<init>(PackedLongValues.java:131)
>>         at org.apache.lucene.util.packed.PackedLongValues.iterator(PackedLongValues.java:120)
>>         at org.apache.lucene.index.SortedDocValuesWriter$BufferedSortedDocValues.<init>(SortedDocValuesWriter.java:194)
>>         at org.apache.lucene.index.SortedDocValuesWriter$1.getSorted(SortedDocValuesWriter.java:171)
>>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:413)
>>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedField(Lucene70DocValuesConsumer.java:394)
>>         at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedField(AssertingDocValuesFormat.java:141)
>>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
>>         at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
>>         at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
>>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
>>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
>>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
>>         at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:416)
>>         at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:517)
>>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1776)
>>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1468)
>>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:185)
>>         at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.lang.reflect.Method.invoke(Method.java:498)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>
>>
>> FAILED:  org.apache.lucene.search.TestCustomSearcherSort.testFieldSortSingleSearcher
>>
>> Error Message:
>> GC overhead limit exceeded
>>
>> Stack Trace:
>> java.lang.OutOfMemoryError: GC overhead limit exceeded
>>         at java.util.Arrays.copyOf(Arrays.java:3332)
>>         at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
>>         at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
>>         at java.lang.StringBuilder.append(StringBuilder.java:136)
>>         at java.lang.StringBuilder.append(StringBuilder.java:131)
>>         at org.apache.lucene.index.IndexWriterConfig.toString(IndexWriterConfig.java:480)
>>         at org.apache.lucene.util.LuceneTestCase.maybeChangeLiveIndexWriterConfig(LuceneTestCase.java:1157)
>>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:148)
>>         at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.lang.reflect.Method.invoke(Method.java:498)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>>
>>
>> FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestCustomSearcherSort
>>
>> Error Message:
>> Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>>
>> Stack Trace:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>>         at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
>> Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>         at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>>         Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
>>                 at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
>>                 at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
>>                 at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1872)
>>                 at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
>>                 at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
>> Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>         at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
>>         at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
>>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
>>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 1880 lines...]
>>    [junit4] Suite: org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSoftDeletesRetentionMergePolicy -Dtests.method=testSoftDeleteWithRetention -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn -Dtests.timezone=Etc/GMT-0 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>>    [junit4] ERROR   3948s J1 | TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention <<<
>>    [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([248C95A59BE17434:3432A38A78701E91]:0)
>>    [junit4]    >        at java.util.HashMap$KeySet.iterator(HashMap.java:917)
>>    [junit4]    >        at java.util.HashSet.iterator(HashSet.java:173)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.maxNumSegmentsMergesPending(IndexWriter.java:2216)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2190)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2119)
>>    [junit4]    >        at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention(TestSoftDeletesRetentionMergePolicy.java:266)
>>    [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J1/temp/lucene.index.TestSoftDeletesRetentionMergePolicy_248C95A59BE17434-001
>>    [junit4]   2> Apr 09, 2018 7:11:50 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>>    [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
>>    [junit4]   2> Apr 09, 2018 7:12:11 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>>    [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:
>>    [junit4]   2>    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>>    [junit4]   2>         at java.lang.Thread.yield(Native Method)
>>    [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
>>    [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
>>    [junit4]   2>         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
>>    [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
>>    [junit4]   2>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
>>    [junit4]   2>         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
>>    [junit4]   2>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
>>    [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
>>    [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
>>    [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
>>    [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
>>    [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>>    [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>>    [junit4]   2> Apr 09, 2018 7:12:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
>>    [junit4]   2> INFO: Starting to interrupt leaked threads:
>>    [junit4]   2>    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=TERMINATED, group={null group}]
>>    [junit4]   2> Apr 09, 2018 7:12:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
>>    [junit4]   2> INFO: All leaked threads terminated.
>>    [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=3, maxDocsPerChunk=139, blockSize=342), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=3, blockSize=342)), sim=RandomSimilarity(queryNorm=false): {}, locale=sr-Latn, timezone=Etc/GMT-0
>>    [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=1315128,total=536346624
>>    [junit4]   2> NOTE: All tests run in this JVM: [TestPriorityQueue, TestCheckIndex, TestCustomNorms, TestFilterLeafReader, TestDoubleRangeFieldQueries, TestExternalCodecs, TestComplexExplanations, TestIndexWriterFromReader, TestMinimize, TestTermVectorsReader, TestAllFilesHaveChecksumFooter, TestRAMDirectory, TestIndexWriterUnicode, TestSimpleAttributeImpl, TestSpanFirstQuery, TestTimeLimitingCollector, TestForTooMuchCloning, Test2BTerms, Test2BPostings, TestByteSlices, TestClassicSimilarity, TestToken, TestQueryRescorer, TestFileSwitchDirectory, TestLucene70DocValuesFormat, TestWeakIdentityMap, TestLucene50StoredFieldsFormat, TestLongValuesSource, TestAutomatonQueryUnicode, TestRollback, TestDemo, TestDemoParallelLeafReader, TestIndexFileDeleter, TestLucene60FieldInfoFormat, TestSpanCollection, TestPolygon2D, TestDeterminism, TestBooleanRewrites, TestRecyclingIntBlockAllocator, TestGeoEncodingUtils, TestIntroSorter, TestWildcard, TestIndexWriterOutOfFileDescriptors, TestFSTs, TestSpanNotQuery, TestRegExp, TestBooleanQuery, TestMultiTermConstantScore, TestFieldReuse, TestRecyclingByteBlockAllocator, TestMutableValues, TestFrequencyTrackingRingBuffer, TestTerms, TestExceedMaxTermLength, TestIndexWriterDelete, TestSizeBoundedForceMerge, TestDocIdSetBuilder, Test2BPostingsBytes, Test2BPagedBytes, TestNativeFSLockFactory, TestEarlyTermination, LimitedFiniteStringsIteratorTest, TestMSBRadixSorter, Test2BDocs, TestNGramPhraseQuery, TestDirectory, TestDocIdsWriter, TestFloatRangeFieldQueries, TestTransactionRollback, TestPendingDeletes, TestControlledRealTimeReopenThread, TestNumericDocValuesUpdates, TestPerFieldDocValuesFormat, TestSoftDeletesRetentionMergePolicy]
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSoftDeletesRetentionMergePolicy -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn -Dtests.timezone=Etc/GMT-0 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>>    [junit4] ERROR   0.00s J1 | TestSoftDeletesRetentionMergePolicy (suite) <<<
>>    [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:
>>    [junit4]    >    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>>    [junit4]    >         at java.lang.Thread.yield(Native Method)
>>    [junit4]    >         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
>>    [junit4]    >         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
>>    [junit4]    >         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
>>    [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
>>    [junit4]    >         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]    >         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
>>    [junit4]    >         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
>>    [junit4]    >         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
>>    [junit4]    >         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
>>    [junit4]    >         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
>>    [junit4]    >         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
>>    [junit4]    >         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
>>    [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>>    [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
>>    [junit4] Completed [466/470 (1!)] on J1 in 3970.86s, 4 tests, 2 errors <<< FAILURES!
>>
>> [...truncated 1 lines...]
>>    [junit4] Suite: org.apache.lucene.search.TestCustomSearcherSort
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.method=testFieldSortCustomSearcher -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>>    [junit4] ERROR   5.35s J1 | TestCustomSearcherSort.testFieldSortCustomSearcher <<<
>>    [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>    [junit4]    >        at org.apache.lucene.util.packed.PackedLongValues$Iterator.<init>(PackedLongValues.java:131)
>>    [junit4]    >        at org.apache.lucene.util.packed.PackedLongValues.iterator(PackedLongValues.java:120)
>>    [junit4]    >        at org.apache.lucene.index.SortedDocValuesWriter$BufferedSortedDocValues.<init>(SortedDocValuesWriter.java:194)
>>    [junit4]    >        at org.apache.lucene.index.SortedDocValuesWriter$1.getSorted(SortedDocValuesWriter.java:171)
>>    [junit4]    >        at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:413)
>>    [junit4]    >        at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedField(Lucene70DocValuesConsumer.java:394)
>>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedField(AssertingDocValuesFormat.java:141)
>>    [junit4]    >        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
>>    [junit4]    >        at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
>>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
>>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
>>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
>>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
>>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:416)
>>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:517)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1776)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1468)
>>    [junit4]    >        at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:185)
>>    [junit4]    >        at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)Throwable #2: java.lang.NullPointerException
>>    [junit4]    >        at org.apache.lucene.search.TestCustomSearcherSort.tearDown(TestCustomSearcherSort.java:79)
>>    [junit4]    >        at java.lang.Thread.run(Thread.java:748)
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.method=testFieldSortSingleSearcher -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>>    [junit4] ERROR   4.00s J1 | TestCustomSearcherSort.testFieldSortSingleSearcher <<<
>>    [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>    [junit4]    >        at java.util.Arrays.copyOf(Arrays.java:3332)
>>    [junit4]    >        at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
>>    [junit4]    >        at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
>>    [junit4]    >        at java.lang.StringBuilder.append(StringBuilder.java:136)
>>    [junit4]    >        at java.lang.StringBuilder.append(StringBuilder.java:131)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriterConfig.toString(IndexWriterConfig.java:480)
>>    [junit4]    >        at org.apache.lucene.util.LuceneTestCase.maybeChangeLiveIndexWriterConfig(LuceneTestCase.java:1157)
>>    [junit4]    >        at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:148)
>>    [junit4]    >        at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)Throwable #2: java.lang.NullPointerException
>>    [junit4]    >        at org.apache.lucene.search.TestCustomSearcherSort.tearDown(TestCustomSearcherSort.java:79)
>>    [junit4]    >        at java.lang.Thread.run(Thread.java:748)
>>    [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J1/temp/lucene.search.TestCustomSearcherSort_248C95A59BE17434-001
>>    [junit4]   2> Apr 09, 2018 7:12:30 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>>    [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
>>    [junit4]   2> Apr 09, 2018 7:12:34 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #281253,5,TGRP-TestSoftDeletesRetentionMergePolicy]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>>    [junit4]   2>        Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
>>    [junit4]   2>                at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
>>    [junit4]   2>                at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
>>    [junit4]   2>                at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1872)
>>    [junit4]   2>                at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
>>    [junit4]   2>                at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
>>    [junit4]   2> Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>    [junit4]   2>        at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>>    [junit4]   2>
>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{publicationDate_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1664, maxMBSortInHeap=7.714473367744617, sim=RandomSimilarity(queryNorm=true): {content=IB SPL-LZ(0.3)}, locale=sr-CS, timezone=Asia/Thimphu
>>    [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=1254800,total=536346624
>>    [junit4]   2> NOTE: All tests run in this JVM: [TestPriorityQueue, TestCheckIndex, TestCustomNorms, TestFilterLeafReader, TestDoubleRangeFieldQueries, TestExternalCodecs, TestComplexExplanations, TestIndexWriterFromReader, TestMinimize, TestTermVectorsReader, TestAllFilesHaveChecksumFooter, TestRAMDirectory, TestIndexWriterUnicode, TestSimpleAttributeImpl, TestSpanFirstQuery, TestTimeLimitingCollector, TestForTooMuchCloning, Test2BTerms, Test2BPostings, TestByteSlices, TestClassicSimilarity, TestToken, TestQueryRescorer, TestFileSwitchDirectory, TestLucene70DocValuesFormat, TestWeakIdentityMap, TestLucene50StoredFieldsFormat, TestLongValuesSource, TestAutomatonQueryUnicode, TestRollback, TestDemo, TestDemoParallelLeafReader, TestIndexFileDeleter, TestLucene60FieldInfoFormat, TestSpanCollection, TestPolygon2D, TestDeterminism, TestBooleanRewrites, TestRecyclingIntBlockAllocator, TestGeoEncodingUtils, TestIntroSorter, TestWildcard, TestIndexWriterOutOfFileDescriptors, TestFSTs, TestSpanNotQuery, TestRegExp, TestBooleanQuery, TestMultiTermConstantScore, TestFieldReuse, TestRecyclingByteBlockAllocator, TestMutableValues, TestFrequencyTrackingRingBuffer, TestTerms, TestExceedMaxTermLength, TestIndexWriterDelete, TestSizeBoundedForceMerge, TestDocIdSetBuilder, Test2BPostingsBytes, Test2BPagedBytes, TestNativeFSLockFactory, TestEarlyTermination, LimitedFiniteStringsIteratorTest, TestMSBRadixSorter, Test2BDocs, TestNGramPhraseQuery, TestDirectory, TestDocIdsWriter, TestFloatRangeFieldQueries, TestTransactionRollback, TestPendingDeletes, TestControlledRealTimeReopenThread, TestNumericDocValuesUpdates, TestPerFieldDocValuesFormat, TestSoftDeletesRetentionMergePolicy, TestCustomSearcherSort]
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>>    [junit4] ERROR   0.00s J1 | TestCustomSearcherSort (suite) <<<
>>    [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
>>    [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>>    [junit4]    >        Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
>>    [junit4]    >                at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
>>    [junit4]    >                at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
>>    [junit4]    >                at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
>>    [junit4]    >                at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
>>    [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
>>    [junit4]    >        at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
>>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>>    [junit4] Completed [467/470 (2!)] on J1 in 18.88s, 2 tests, 3 errors <<< FAILURES!
>>
>> [...truncated 10 lines...]
>>    [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J1-20180409_052828_757440584628941507603.sysout
>>    [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
>>    [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
>>    [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid17525.hprof ...
>>    [junit4] Heap dump file created [803738033 bytes in 4.637 secs]
>>    [junit4] <<< JVM J1: EOF ----
>>
>> [...truncated 22795 lines...]
>> BUILD FAILED
>> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
>> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
>> * java_pid17525.hprof
>>
>> Total time: 278 minutes 11 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Archiving artifacts
>> Recording test results
>> Email was triggered for: Failure - Any
>> Sending email for trigger: Failure - Any
>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 196 - Failure

Posted by Simon Willnauer <si...@gmail.com>.
I am looking into this

On Mon, Apr 9, 2018 at 12:06 PM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/196/
>
> 5 tests failed.
> FAILED:  org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention
>
> Error Message:
> GC overhead limit exceeded
>
> Stack Trace:
> java.lang.OutOfMemoryError: GC overhead limit exceeded
>         at __randomizedtesting.SeedInfo.seed([248C95A59BE17434:3432A38A78701E91]:0)
>         at java.util.HashMap$KeySet.iterator(HashMap.java:917)
>         at java.util.HashSet.iterator(HashSet.java:173)
>         at org.apache.lucene.index.IndexWriter.maxNumSegmentsMergesPending(IndexWriter.java:2216)
>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2190)
>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2119)
>         at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention(TestSoftDeletesRetentionMergePolicy.java:266)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>
>
> FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy
>
> Error Message:
> 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:     1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]         at java.lang.Thread.yield(Native Method)         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:
>    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>         at java.lang.Thread.yield(Native Method)
>         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
>         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
>         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
>         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
>         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>         at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
>
>
> FAILED:  org.apache.lucene.search.TestCustomSearcherSort.testFieldSortCustomSearcher
>
> Error Message:
> GC overhead limit exceeded
>
> Stack Trace:
> java.lang.OutOfMemoryError: GC overhead limit exceeded
>         at org.apache.lucene.util.packed.PackedLongValues$Iterator.<init>(PackedLongValues.java:131)
>         at org.apache.lucene.util.packed.PackedLongValues.iterator(PackedLongValues.java:120)
>         at org.apache.lucene.index.SortedDocValuesWriter$BufferedSortedDocValues.<init>(SortedDocValuesWriter.java:194)
>         at org.apache.lucene.index.SortedDocValuesWriter$1.getSorted(SortedDocValuesWriter.java:171)
>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:413)
>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedField(Lucene70DocValuesConsumer.java:394)
>         at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedField(AssertingDocValuesFormat.java:141)
>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
>         at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
>         at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
>         at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:416)
>         at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:517)
>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1776)
>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1468)
>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:185)
>         at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>
>
> FAILED:  org.apache.lucene.search.TestCustomSearcherSort.testFieldSortSingleSearcher
>
> Error Message:
> GC overhead limit exceeded
>
> Stack Trace:
> java.lang.OutOfMemoryError: GC overhead limit exceeded
>         at java.util.Arrays.copyOf(Arrays.java:3332)
>         at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
>         at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
>         at java.lang.StringBuilder.append(StringBuilder.java:136)
>         at java.lang.StringBuilder.append(StringBuilder.java:131)
>         at org.apache.lucene.index.IndexWriterConfig.toString(IndexWriterConfig.java:480)
>         at org.apache.lucene.util.LuceneTestCase.maybeChangeLiveIndexWriterConfig(LuceneTestCase.java:1157)
>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:148)
>         at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>
>
> FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestCustomSearcherSort
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>         at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
> Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
>         at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>         Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
>                 at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
>                 at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
>                 at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1872)
>                 at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
>                 at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
> Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
>         at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
>         at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>
>
>
>
> Build Log:
> [...truncated 1880 lines...]
>    [junit4] Suite: org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSoftDeletesRetentionMergePolicy -Dtests.method=testSoftDeleteWithRetention -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn -Dtests.timezone=Etc/GMT-0 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>    [junit4] ERROR   3948s J1 | TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention <<<
>    [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([248C95A59BE17434:3432A38A78701E91]:0)
>    [junit4]    >        at java.util.HashMap$KeySet.iterator(HashMap.java:917)
>    [junit4]    >        at java.util.HashSet.iterator(HashSet.java:173)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.maxNumSegmentsMergesPending(IndexWriter.java:2216)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2190)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2119)
>    [junit4]    >        at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention(TestSoftDeletesRetentionMergePolicy.java:266)
>    [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J1/temp/lucene.index.TestSoftDeletesRetentionMergePolicy_248C95A59BE17434-001
>    [junit4]   2> Apr 09, 2018 7:11:50 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>    [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
>    [junit4]   2> Apr 09, 2018 7:12:11 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>    [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:
>    [junit4]   2>    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>    [junit4]   2>         at java.lang.Thread.yield(Native Method)
>    [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
>    [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
>    [junit4]   2>         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
>    [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
>    [junit4]   2>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
>    [junit4]   2>         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
>    [junit4]   2>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
>    [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
>    [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
>    [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
>    [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
>    [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>    [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>    [junit4]   2> Apr 09, 2018 7:12:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
>    [junit4]   2> INFO: Starting to interrupt leaked threads:
>    [junit4]   2>    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=TERMINATED, group={null group}]
>    [junit4]   2> Apr 09, 2018 7:12:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
>    [junit4]   2> INFO: All leaked threads terminated.
>    [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=3, maxDocsPerChunk=139, blockSize=342), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=3, blockSize=342)), sim=RandomSimilarity(queryNorm=false): {}, locale=sr-Latn, timezone=Etc/GMT-0
>    [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=1315128,total=536346624
>    [junit4]   2> NOTE: All tests run in this JVM: [TestPriorityQueue, TestCheckIndex, TestCustomNorms, TestFilterLeafReader, TestDoubleRangeFieldQueries, TestExternalCodecs, TestComplexExplanations, TestIndexWriterFromReader, TestMinimize, TestTermVectorsReader, TestAllFilesHaveChecksumFooter, TestRAMDirectory, TestIndexWriterUnicode, TestSimpleAttributeImpl, TestSpanFirstQuery, TestTimeLimitingCollector, TestForTooMuchCloning, Test2BTerms, Test2BPostings, TestByteSlices, TestClassicSimilarity, TestToken, TestQueryRescorer, TestFileSwitchDirectory, TestLucene70DocValuesFormat, TestWeakIdentityMap, TestLucene50StoredFieldsFormat, TestLongValuesSource, TestAutomatonQueryUnicode, TestRollback, TestDemo, TestDemoParallelLeafReader, TestIndexFileDeleter, TestLucene60FieldInfoFormat, TestSpanCollection, TestPolygon2D, TestDeterminism, TestBooleanRewrites, TestRecyclingIntBlockAllocator, TestGeoEncodingUtils, TestIntroSorter, TestWildcard, TestIndexWriterOutOfFileDescriptors, TestFSTs, TestSpanNotQuery, TestRegExp, TestBooleanQuery, TestMultiTermConstantScore, TestFieldReuse, TestRecyclingByteBlockAllocator, TestMutableValues, TestFrequencyTrackingRingBuffer, TestTerms, TestExceedMaxTermLength, TestIndexWriterDelete, TestSizeBoundedForceMerge, TestDocIdSetBuilder, Test2BPostingsBytes, Test2BPagedBytes, TestNativeFSLockFactory, TestEarlyTermination, LimitedFiniteStringsIteratorTest, TestMSBRadixSorter, Test2BDocs, TestNGramPhraseQuery, TestDirectory, TestDocIdsWriter, TestFloatRangeFieldQueries, TestTransactionRollback, TestPendingDeletes, TestControlledRealTimeReopenThread, TestNumericDocValuesUpdates, TestPerFieldDocValuesFormat, TestSoftDeletesRetentionMergePolicy]
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSoftDeletesRetentionMergePolicy -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn -Dtests.timezone=Etc/GMT-0 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>    [junit4] ERROR   0.00s J1 | TestSoftDeletesRetentionMergePolicy (suite) <<<
>    [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:
>    [junit4]    >    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>    [junit4]    >         at java.lang.Thread.yield(Native Method)
>    [junit4]    >         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
>    [junit4]    >         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
>    [junit4]    >         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
>    [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
>    [junit4]    >         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]    >         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
>    [junit4]    >         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
>    [junit4]    >         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
>    [junit4]    >         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
>    [junit4]    >         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
>    [junit4]    >         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
>    [junit4]    >         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
>    [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>    [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
>    [junit4] Completed [466/470 (1!)] on J1 in 3970.86s, 4 tests, 2 errors <<< FAILURES!
>
> [...truncated 1 lines...]
>    [junit4] Suite: org.apache.lucene.search.TestCustomSearcherSort
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.method=testFieldSortCustomSearcher -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>    [junit4] ERROR   5.35s J1 | TestCustomSearcherSort.testFieldSortCustomSearcher <<<
>    [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4]    >        at org.apache.lucene.util.packed.PackedLongValues$Iterator.<init>(PackedLongValues.java:131)
>    [junit4]    >        at org.apache.lucene.util.packed.PackedLongValues.iterator(PackedLongValues.java:120)
>    [junit4]    >        at org.apache.lucene.index.SortedDocValuesWriter$BufferedSortedDocValues.<init>(SortedDocValuesWriter.java:194)
>    [junit4]    >        at org.apache.lucene.index.SortedDocValuesWriter$1.getSorted(SortedDocValuesWriter.java:171)
>    [junit4]    >        at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:413)
>    [junit4]    >        at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedField(Lucene70DocValuesConsumer.java:394)
>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedField(AssertingDocValuesFormat.java:141)
>    [junit4]    >        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
>    [junit4]    >        at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:416)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:517)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1776)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1468)
>    [junit4]    >        at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:185)
>    [junit4]    >        at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)Throwable #2: java.lang.NullPointerException
>    [junit4]    >        at org.apache.lucene.search.TestCustomSearcherSort.tearDown(TestCustomSearcherSort.java:79)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:748)
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.method=testFieldSortSingleSearcher -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>    [junit4] ERROR   4.00s J1 | TestCustomSearcherSort.testFieldSortSingleSearcher <<<
>    [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4]    >        at java.util.Arrays.copyOf(Arrays.java:3332)
>    [junit4]    >        at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
>    [junit4]    >        at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
>    [junit4]    >        at java.lang.StringBuilder.append(StringBuilder.java:136)
>    [junit4]    >        at java.lang.StringBuilder.append(StringBuilder.java:131)
>    [junit4]    >        at org.apache.lucene.index.IndexWriterConfig.toString(IndexWriterConfig.java:480)
>    [junit4]    >        at org.apache.lucene.util.LuceneTestCase.maybeChangeLiveIndexWriterConfig(LuceneTestCase.java:1157)
>    [junit4]    >        at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:148)
>    [junit4]    >        at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)Throwable #2: java.lang.NullPointerException
>    [junit4]    >        at org.apache.lucene.search.TestCustomSearcherSort.tearDown(TestCustomSearcherSort.java:79)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:748)
>    [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J1/temp/lucene.search.TestCustomSearcherSort_248C95A59BE17434-001
>    [junit4]   2> Apr 09, 2018 7:12:30 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>    [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
>    [junit4]   2> Apr 09, 2018 7:12:34 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #281253,5,TGRP-TestSoftDeletesRetentionMergePolicy]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>    [junit4]   2>        Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
>    [junit4]   2>                at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
>    [junit4]   2>                at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
>    [junit4]   2>                at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1872)
>    [junit4]   2>                at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
>    [junit4]   2>                at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
>    [junit4]   2> Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4]   2>        at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>    [junit4]   2>
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{publicationDate_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1664, maxMBSortInHeap=7.714473367744617, sim=RandomSimilarity(queryNorm=true): {content=IB SPL-LZ(0.3)}, locale=sr-CS, timezone=Asia/Thimphu
>    [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=1254800,total=536346624
>    [junit4]   2> NOTE: All tests run in this JVM: [TestPriorityQueue, TestCheckIndex, TestCustomNorms, TestFilterLeafReader, TestDoubleRangeFieldQueries, TestExternalCodecs, TestComplexExplanations, TestIndexWriterFromReader, TestMinimize, TestTermVectorsReader, TestAllFilesHaveChecksumFooter, TestRAMDirectory, TestIndexWriterUnicode, TestSimpleAttributeImpl, TestSpanFirstQuery, TestTimeLimitingCollector, TestForTooMuchCloning, Test2BTerms, Test2BPostings, TestByteSlices, TestClassicSimilarity, TestToken, TestQueryRescorer, TestFileSwitchDirectory, TestLucene70DocValuesFormat, TestWeakIdentityMap, TestLucene50StoredFieldsFormat, TestLongValuesSource, TestAutomatonQueryUnicode, TestRollback, TestDemo, TestDemoParallelLeafReader, TestIndexFileDeleter, TestLucene60FieldInfoFormat, TestSpanCollection, TestPolygon2D, TestDeterminism, TestBooleanRewrites, TestRecyclingIntBlockAllocator, TestGeoEncodingUtils, TestIntroSorter, TestWildcard, TestIndexWriterOutOfFileDescriptors, TestFSTs, TestSpanNotQuery, TestRegExp, TestBooleanQuery, TestMultiTermConstantScore, TestFieldReuse, TestRecyclingByteBlockAllocator, TestMutableValues, TestFrequencyTrackingRingBuffer, TestTerms, TestExceedMaxTermLength, TestIndexWriterDelete, TestSizeBoundedForceMerge, TestDocIdSetBuilder, Test2BPostingsBytes, Test2BPagedBytes, TestNativeFSLockFactory, TestEarlyTermination, LimitedFiniteStringsIteratorTest, TestMSBRadixSorter, Test2BDocs, TestNGramPhraseQuery, TestDirectory, TestDocIdsWriter, TestFloatRangeFieldQueries, TestTransactionRollback, TestPendingDeletes, TestControlledRealTimeReopenThread, TestNumericDocValuesUpdates, TestPerFieldDocValuesFormat, TestSoftDeletesRetentionMergePolicy, TestCustomSearcherSort]
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>    [junit4] ERROR   0.00s J1 | TestCustomSearcherSort (suite) <<<
>    [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
>    [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>    [junit4]    >        Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
>    [junit4]    >                at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
>    [junit4]    >                at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
>    [junit4]    >                at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
>    [junit4]    >                at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
>    [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4]    >        at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>    [junit4] Completed [467/470 (2!)] on J1 in 18.88s, 2 tests, 3 errors <<< FAILURES!
>
> [...truncated 10 lines...]
>    [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J1-20180409_052828_757440584628941507603.sysout
>    [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
>    [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid17525.hprof ...
>    [junit4] Heap dump file created [803738033 bytes in 4.637 secs]
>    [junit4] <<< JVM J1: EOF ----
>
> [...truncated 22795 lines...]
> BUILD FAILED
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
> * java_pid17525.hprof
>
> Total time: 278 minutes 11 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 196 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/196/

5 tests failed.
FAILED:  org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([248C95A59BE17434:3432A38A78701E91]:0)
	at java.util.HashMap$KeySet.iterator(HashMap.java:917)
	at java.util.HashSet.iterator(HashSet.java:173)
	at org.apache.lucene.index.IndexWriter.maxNumSegmentsMergesPending(IndexWriter.java:2216)
	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2190)
	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2119)
	at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention(TestSoftDeletesRetentionMergePolicy.java:266)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy

Error Message:
2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy:     1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]         at java.lang.Thread.yield(Native Method)         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy: 
   1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
        at java.lang.Thread.yield(Native Method)
        at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
        at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
        at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
        at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
        at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
        at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
        at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
	at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)


FAILED:  org.apache.lucene.search.TestCustomSearcherSort.testFieldSortCustomSearcher

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at org.apache.lucene.util.packed.PackedLongValues$Iterator.<init>(PackedLongValues.java:131)
	at org.apache.lucene.util.packed.PackedLongValues.iterator(PackedLongValues.java:120)
	at org.apache.lucene.index.SortedDocValuesWriter$BufferedSortedDocValues.<init>(SortedDocValuesWriter.java:194)
	at org.apache.lucene.index.SortedDocValuesWriter$1.getSorted(SortedDocValuesWriter.java:171)
	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:413)
	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedField(Lucene70DocValuesConsumer.java:394)
	at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedField(AssertingDocValuesFormat.java:141)
	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
	at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
	at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:416)
	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:517)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1776)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1468)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:185)
	at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)


FAILED:  org.apache.lucene.search.TestCustomSearcherSort.testFieldSortSingleSearcher

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at java.util.Arrays.copyOf(Arrays.java:3332)
	at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
	at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
	at java.lang.StringBuilder.append(StringBuilder.java:136)
	at java.lang.StringBuilder.append(StringBuilder.java:131)
	at org.apache.lucene.index.IndexWriterConfig.toString(IndexWriterConfig.java:480)
	at org.apache.lucene.util.LuceneTestCase.maybeChangeLiveIndexWriterConfig(LuceneTestCase.java:1157)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:148)
	at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestCustomSearcherSort

Error Message:
Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
	at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
	Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
		at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
		at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
		at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1872)
		at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
		at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
	at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)




Build Log:
[...truncated 1880 lines...]
   [junit4] Suite: org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSoftDeletesRetentionMergePolicy -Dtests.method=testSoftDeleteWithRetention -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn -Dtests.timezone=Etc/GMT-0 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   3948s J1 | TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([248C95A59BE17434:3432A38A78701E91]:0)
   [junit4]    > 	at java.util.HashMap$KeySet.iterator(HashMap.java:917)
   [junit4]    > 	at java.util.HashSet.iterator(HashSet.java:173)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.maxNumSegmentsMergesPending(IndexWriter.java:2216)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2190)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2119)
   [junit4]    > 	at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy.testSoftDeleteWithRetention(TestSoftDeletesRetentionMergePolicy.java:266)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J1/temp/lucene.index.TestSoftDeletesRetentionMergePolicy_248C95A59BE17434-001
   [junit4]   2> Apr 09, 2018 7:11:50 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Apr 09, 2018 7:12:11 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy: 
   [junit4]   2>    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
   [junit4]   2>         at java.lang.Thread.yield(Native Method)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
   [junit4]   2>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
   [junit4]   2>         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
   [junit4]   2>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
   [junit4]   2>         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
   [junit4]   2>         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> Apr 09, 2018 7:12:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=TERMINATED, group={null group}]
   [junit4]   2> Apr 09, 2018 7:12:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=3, maxDocsPerChunk=139, blockSize=342), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=3, blockSize=342)), sim=RandomSimilarity(queryNorm=false): {}, locale=sr-Latn, timezone=Etc/GMT-0
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=1315128,total=536346624
   [junit4]   2> NOTE: All tests run in this JVM: [TestPriorityQueue, TestCheckIndex, TestCustomNorms, TestFilterLeafReader, TestDoubleRangeFieldQueries, TestExternalCodecs, TestComplexExplanations, TestIndexWriterFromReader, TestMinimize, TestTermVectorsReader, TestAllFilesHaveChecksumFooter, TestRAMDirectory, TestIndexWriterUnicode, TestSimpleAttributeImpl, TestSpanFirstQuery, TestTimeLimitingCollector, TestForTooMuchCloning, Test2BTerms, Test2BPostings, TestByteSlices, TestClassicSimilarity, TestToken, TestQueryRescorer, TestFileSwitchDirectory, TestLucene70DocValuesFormat, TestWeakIdentityMap, TestLucene50StoredFieldsFormat, TestLongValuesSource, TestAutomatonQueryUnicode, TestRollback, TestDemo, TestDemoParallelLeafReader, TestIndexFileDeleter, TestLucene60FieldInfoFormat, TestSpanCollection, TestPolygon2D, TestDeterminism, TestBooleanRewrites, TestRecyclingIntBlockAllocator, TestGeoEncodingUtils, TestIntroSorter, TestWildcard, TestIndexWriterOutOfFileDescriptors, TestFSTs, TestSpanNotQuery, TestRegExp, TestBooleanQuery, TestMultiTermConstantScore, TestFieldReuse, TestRecyclingByteBlockAllocator, TestMutableValues, TestFrequencyTrackingRingBuffer, TestTerms, TestExceedMaxTermLength, TestIndexWriterDelete, TestSizeBoundedForceMerge, TestDocIdSetBuilder, Test2BPostingsBytes, Test2BPagedBytes, TestNativeFSLockFactory, TestEarlyTermination, LimitedFiniteStringsIteratorTest, TestMSBRadixSorter, Test2BDocs, TestNGramPhraseQuery, TestDirectory, TestDocIdsWriter, TestFloatRangeFieldQueries, TestTransactionRollback, TestPendingDeletes, TestControlledRealTimeReopenThread, TestNumericDocValuesUpdates, TestPerFieldDocValuesFormat, TestSoftDeletesRetentionMergePolicy]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSoftDeletesRetentionMergePolicy -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn -Dtests.timezone=Etc/GMT-0 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestSoftDeletesRetentionMergePolicy (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.lucene.index.TestSoftDeletesRetentionMergePolicy: 
   [junit4]    >    1) Thread[id=289762, name=Lucene Merge Thread #281215, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
   [junit4]    >         at java.lang.Thread.yield(Native Method)
   [junit4]    >         at org.apache.lucene.store.MockDirectoryWrapper.maybeYield(MockDirectoryWrapper.java:626)
   [junit4]    >         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:644)
   [junit4]    >         at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
   [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler$1.createOutput(ConcurrentMergeScheduler.java:288)
   [junit4]    >         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    >         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.finish(Lucene60PointsWriter.java:243)
   [junit4]    >         at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:189)
   [junit4]    >         at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:144)
   [junit4]    >         at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:187)
   [junit4]    >         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:136)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4483)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144)
   [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]    >         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
   [junit4] Completed [466/470 (1!)] on J1 in 3970.86s, 4 tests, 2 errors <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.lucene.search.TestCustomSearcherSort
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.method=testFieldSortCustomSearcher -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   5.35s J1 | TestCustomSearcherSort.testFieldSortCustomSearcher <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at org.apache.lucene.util.packed.PackedLongValues$Iterator.<init>(PackedLongValues.java:131)
   [junit4]    > 	at org.apache.lucene.util.packed.PackedLongValues.iterator(PackedLongValues.java:120)
   [junit4]    > 	at org.apache.lucene.index.SortedDocValuesWriter$BufferedSortedDocValues.<init>(SortedDocValuesWriter.java:194)
   [junit4]    > 	at org.apache.lucene.index.SortedDocValuesWriter$1.getSorted(SortedDocValuesWriter.java:171)
   [junit4]    > 	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:413)
   [junit4]    > 	at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedField(Lucene70DocValuesConsumer.java:394)
   [junit4]    > 	at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedField(AssertingDocValuesFormat.java:141)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedField(PerFieldDocValuesFormat.java:119)
   [junit4]    > 	at org.apache.lucene.index.SortedDocValuesWriter.flush(SortedDocValuesWriter.java:163)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:248)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:132)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:416)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:517)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1776)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1468)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:185)
   [junit4]    > 	at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)Throwable #2: java.lang.NullPointerException
   [junit4]    > 	at org.apache.lucene.search.TestCustomSearcherSort.tearDown(TestCustomSearcherSort.java:79)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.method=testFieldSortSingleSearcher -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   4.00s J1 | TestCustomSearcherSort.testFieldSortSingleSearcher <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at java.util.Arrays.copyOf(Arrays.java:3332)
   [junit4]    > 	at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
   [junit4]    > 	at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
   [junit4]    > 	at java.lang.StringBuilder.append(StringBuilder.java:136)
   [junit4]    > 	at java.lang.StringBuilder.append(StringBuilder.java:131)
   [junit4]    > 	at org.apache.lucene.index.IndexWriterConfig.toString(IndexWriterConfig.java:480)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.maybeChangeLiveIndexWriterConfig(LuceneTestCase.java:1157)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:148)
   [junit4]    > 	at org.apache.lucene.search.TestCustomSearcherSort.setUp(TestCustomSearcherSort.java:70)Throwable #2: java.lang.NullPointerException
   [junit4]    > 	at org.apache.lucene.search.TestCustomSearcherSort.tearDown(TestCustomSearcherSort.java:79)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J1/temp/lucene.search.TestCustomSearcherSort_248C95A59BE17434-001
   [junit4]   2> Apr 09, 2018 7:12:30 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Apr 09, 2018 7:12:34 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #281253,5,TGRP-TestSoftDeletesRetentionMergePolicy]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]   2> 	Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner.augmentStackTrace(RandomizedRunner.java:1872)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
   [junit4]   2> 		at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]   2> 	at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{publicationDate_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1664, maxMBSortInHeap=7.714473367744617, sim=RandomSimilarity(queryNorm=true): {content=IB SPL-LZ(0.3)}, locale=sr-CS, timezone=Asia/Thimphu
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=1254800,total=536346624
   [junit4]   2> NOTE: All tests run in this JVM: [TestPriorityQueue, TestCheckIndex, TestCustomNorms, TestFilterLeafReader, TestDoubleRangeFieldQueries, TestExternalCodecs, TestComplexExplanations, TestIndexWriterFromReader, TestMinimize, TestTermVectorsReader, TestAllFilesHaveChecksumFooter, TestRAMDirectory, TestIndexWriterUnicode, TestSimpleAttributeImpl, TestSpanFirstQuery, TestTimeLimitingCollector, TestForTooMuchCloning, Test2BTerms, Test2BPostings, TestByteSlices, TestClassicSimilarity, TestToken, TestQueryRescorer, TestFileSwitchDirectory, TestLucene70DocValuesFormat, TestWeakIdentityMap, TestLucene50StoredFieldsFormat, TestLongValuesSource, TestAutomatonQueryUnicode, TestRollback, TestDemo, TestDemoParallelLeafReader, TestIndexFileDeleter, TestLucene60FieldInfoFormat, TestSpanCollection, TestPolygon2D, TestDeterminism, TestBooleanRewrites, TestRecyclingIntBlockAllocator, TestGeoEncodingUtils, TestIntroSorter, TestWildcard, TestIndexWriterOutOfFileDescriptors, TestFSTs, TestSpanNotQuery, TestRegExp, TestBooleanQuery, TestMultiTermConstantScore, TestFieldReuse, TestRecyclingByteBlockAllocator, TestMutableValues, TestFrequencyTrackingRingBuffer, TestTerms, TestExceedMaxTermLength, TestIndexWriterDelete, TestSizeBoundedForceMerge, TestDocIdSetBuilder, Test2BPostingsBytes, Test2BPagedBytes, TestNativeFSLockFactory, TestEarlyTermination, LimitedFiniteStringsIteratorTest, TestMSBRadixSorter, Test2BDocs, TestNGramPhraseQuery, TestDirectory, TestDocIdsWriter, TestFloatRangeFieldQueries, TestTransactionRollback, TestPendingDeletes, TestControlledRealTimeReopenThread, TestNumericDocValuesUpdates, TestPerFieldDocValuesFormat, TestSoftDeletesRetentionMergePolicy, TestCustomSearcherSort]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCustomSearcherSort -Dtests.seed=248C95A59BE17434 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=sr-CS -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestCustomSearcherSort (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([248C95A59BE17434]:0)
   [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
   [junit4]    > 	Suppressed: java.lang.IllegalStateException: No context information for thread: Thread[id=289802, name=Lucene Merge Thread #281253, state=RUNNABLE, group=TGRP-TestSoftDeletesRetentionMergePolicy]. Is this thread running under a class com.carrotsearch.randomizedtesting.RandomizedRunner runner context? Add @RunWith(class com.carrotsearch.randomizedtesting.RandomizedRunner.class) to your test class. Make sure your code accesses random contexts within @BeforeClass and @AfterClass boundary (for example, static test class initializers are not permitted to access random contexts).
   [junit4]    > 		at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:248)
   [junit4]    > 		at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
   [junit4]    > 		at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:20)
   [junit4]    > 		at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at org.apache.lucene.index.BufferedUpdatesStream.waitApplyForMerge(BufferedUpdatesStream.java:202)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeInit(IndexWriter.java:4282)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
   [junit4] Completed [467/470 (2!)] on J1 in 18.88s, 2 tests, 3 errors <<< FAILURES!

[...truncated 10 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J1-20180409_052828_757440584628941507603.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid17525.hprof ...
   [junit4] Heap dump file created [803738033 bytes in 4.637 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 22795 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid17525.hprof

Total time: 278 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 195 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/195/

2 tests failed.
FAILED:  org.apache.lucene.codecs.perfield.TestPerFieldDocValuesFormat.testSparseBinaryFixedLengthVsStoredFields

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([617399315453E295]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.codecs.perfield.TestPerFieldDocValuesFormat

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([617399315453E295]:0)




Build Log:
[...truncated 1659 lines...]
   [junit4] Suite: org.apache.lucene.codecs.perfield.TestPerFieldDocValuesFormat
   [junit4]   2> نيسان 08, 2018 11:03:45 ص com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.codecs.perfield.TestPerFieldDocValuesFormat
   [junit4]   2>    1) Thread[id=11, name=SUITE-TestPerFieldDocValuesFormat-seed#[617399315453E295], state=RUNNABLE, group=TGRP-TestPerFieldDocValuesFormat]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    3) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    4) Thread[id=12, name=TEST-TestPerFieldDocValuesFormat.testSparseBinaryFixedLengthVsStoredFields-seed#[617399315453E295], state=TIMED_WAITING, group=TGRP-TestPerFieldDocValuesFormat]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doWait(IndexWriter.java:4716)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2191)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2119)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.forceMerge(RandomIndexWriter.java:445)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestBinaryVsStoredFields(BaseDocValuesFormatTestCase.java:1472)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestBinaryFixedLengthVsStoredFields(BaseDocValuesFormatTestCase.java:1508)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.testSparseBinaryFixedLengthVsStoredFields(BaseDocValuesFormatTestCase.java:1501)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestPerFieldDocValuesFormat -Dtests.method=testSparseBinaryFixedLengthVsStoredFields -Dtests.seed=617399315453E295 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-SY -Dtests.timezone=Etc/GMT+1 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7068s J1 | TestPerFieldDocValuesFormat.testSparseBinaryFixedLengthVsStoredFields <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([617399315453E295]:0)
   [junit4]   2> نيسان 08, 2018 11:03:45 ص com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> نيسان 08, 2018 11:04:05 ص com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.codecs.perfield.TestPerFieldDocValuesFormat: 
   [junit4]   2>    1) Thread[id=12, name=TEST-TestPerFieldDocValuesFormat.testSparseBinaryFixedLengthVsStoredFields-seed#[617399315453E295], state=TIMED_WAITING, group=TGRP-TestPerFieldDocValuesFormat]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doWait(IndexWriter.java:4716)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2191)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2119)
   [junit4]   2>         at org.apache.lucene.index.RandomIndexWriter.forceMerge(RandomIndexWriter.java:445)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestBinaryVsStoredFields(BaseDocValuesFormatTestCase.java:1472)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestBinaryFixedLengthVsStoredFields(BaseDocValuesFormatTestCase.java:1508)
   [junit4]   2>         at org.apache.lucene.index.BaseDocValuesFormatTestCase.testSparseBinaryFixedLengthVsStoredFields(BaseDocValuesFormatTestCase.java:1501)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> نيسان 08, 2018 11:04:05 ص com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12, name=TEST-TestPerFieldDocValuesFormat.testSparseBinaryFixedLengthVsStoredFields-seed#[617399315453E295], state=TIMED_WAITING, group=TGRP-TestPerFieldDocValuesFormat]
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J1/temp/lucene.codecs.perfield.TestPerFieldDocValuesFormat_617399315453E295-001
   [junit4]   2> Apr 08, 2018 12:04:05 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1220, maxMBSortInHeap=6.77499218789753, sim=RandomSimilarity(queryNorm=true): {fieldname=DFR GLZ(0.3)}, locale=ar-SY, timezone=Etc/GMT+1
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=166666680,total=276824064
   [junit4]   2> NOTE: All tests run in this JVM: [TestPerFieldDocValuesFormat]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestPerFieldDocValuesFormat -Dtests.seed=617399315453E295 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-SY -Dtests.timezone=Etc/GMT+1 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestPerFieldDocValuesFormat (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([617399315453E295]:0)
   [junit4] Completed [393/470 (1!)] on J1 in 7220.66s, 58 tests, 2 errors <<< FAILURES!

[...truncated 62977 lines...]
[asciidoctor:convert] asciidoctor: ERROR: about-this-guide.adoc: line 1: invalid part, must have at least one section (e.g., chapter, appendix, etc.)
[asciidoctor:convert] asciidoctor: ERROR: solr-glossary.adoc: line 1: invalid part, must have at least one section (e.g., chapter, appendix, etc.)
     [java] Processed 2178 links (1735 relative) to 2887 anchors in 227 files
     [echo] Validated Links & Anchors via: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-ref-guide/bare-bones-html/

-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [echo] Checking for malformed docs...

jar-checksums:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

common.resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null144906751
     [copy] Copying 40 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null144906751
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null144906751

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null498758527
     [copy] Copying 246 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null498758527
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null498758527

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 69ms :: artifacts dl 6ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 313 minutes 46 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 194 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/194/

8 tests failed.
FAILED:  org.apache.solr.TestDistributedSearch.test

Error Message:
Captured an uncaught exception in thread: Thread[id=57314, name=Thread-50886, state=RUNNABLE, group=TGRP-TestDistributedSearch]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=57314, name=Thread-50886, state=RUNNABLE, group=TGRP-TestDistributedSearch]
	at __randomizedtesting.SeedInfo.seed([94F000C2E7ADD4A4:1CA43F184951B95C]:0)
Caused by: java.lang.AssertionError: Expected to find shardAddress in the up shard info: {error=org.apache.solr.client.solrj.SolrServerException: Time allowed to handle this request exceeded,trace=org.apache.solr.client.solrj.SolrServerException: Time allowed to handle this request exceeded
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:460)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.makeLoadBalancedRequest(HttpShardHandlerFactory.java:275)
	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:175)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
,time=28}
	at __randomizedtesting.SeedInfo.seed([94F000C2E7ADD4A4]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.TestDistributedSearch.comparePartialResponses(TestDistributedSearch.java:1191)
	at org.apache.solr.TestDistributedSearch$1.run(TestDistributedSearch.java:1147)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:39697/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:39697/collection1
	at __randomizedtesting.SeedInfo.seed([94F000C2E7ADD4A4:1CA43F184951B95C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1591)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:212)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 53 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=61359, name=zkConnectionManagerCallback-7929-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=61358, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[94F000C2E7ADD4A4]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    3) Thread[id=61356, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=61357, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[94F000C2E7ADD4A4]-SendThread(127.0.0.1:44476), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=61359, name=zkConnectionManagerCallback-7929-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=61358, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[94F000C2E7ADD4A4]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   3) Thread[id=61356, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=61357, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[94F000C2E7ADD4A4]-SendThread(127.0.0.1:44476), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
	at __randomizedtesting.SeedInfo.seed([94F000C2E7ADD4A4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=61359, name=zkConnectionManagerCallback-7929-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=61357, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[94F000C2E7ADD4A4]-SendThread(127.0.0.1:44476), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=61359, name=zkConnectionManagerCallback-7929-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=61357, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[94F000C2E7ADD4A4]-SendThread(127.0.0.1:44476), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
	at __randomizedtesting.SeedInfo.seed([94F000C2E7ADD4A4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.security.hadoop.TestSolrCloudWithHadoopAuthPlugin

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
	at __randomizedtesting.SeedInfo.seed([94F000C2E7ADD4A4]:0)
	at sun.nio.ch.Net.bind0(Native Method)
	at sun.nio.ch.Net.bind(Net.java:433)
	at sun.nio.ch.Net.bind(Net.java:425)
	at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
	at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
	at org.apache.mina.transport.socket.nio.NioSocketAcceptor.open(NioSocketAcceptor.java:252)
	at org.apache.mina.transport.socket.nio.NioSocketAcceptor.open(NioSocketAcceptor.java:49)
	at org.apache.mina.core.polling.AbstractPollingIoAcceptor.registerHandles(AbstractPollingIoAcceptor.java:525)
	at org.apache.mina.core.polling.AbstractPollingIoAcceptor.access$200(AbstractPollingIoAcceptor.java:67)
	at org.apache.mina.core.polling.AbstractPollingIoAcceptor$Acceptor.run(AbstractPollingIoAcceptor.java:409)
	at org.apache.mina.util.NamePreservingRunnable.run(NamePreservingRunnable.java:65)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.testParallelExecutorStream

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([E5AC5A05DB47924:B34DB0B964984479]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.testParallelExecutorStream(StreamDecoratorTest.java:3529)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamDecoratorTest

Error Message:
6 threads leaked from SUITE scope at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest:     1) Thread[id=2664, name=zkConnectionManagerCallback-888-thread-1-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    2) Thread[id=2047, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=2663, name=zkConnectionManagerCallback-888-thread-1-SendThread(127.0.0.1:44283), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)    4) Thread[id=2048, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=2044, name=zkConnectionManagerCallback-888-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=2041, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest: 
   1) Thread[id=2664, name=zkConnectionManagerCallback-888-thread-1-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   2) Thread[id=2047, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=2663, name=zkConnectionManagerCallback-888-thread-1-SendThread(127.0.0.1:44283), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
   4) Thread[id=2048, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=2044, name=zkConnectionManagerCallback-888-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=2041, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([E5AC5A05DB47924]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamDecoratorTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=2663, name=zkConnectionManagerCallback-888-thread-1-SendThread(127.0.0.1:44283), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    2) Thread[id=2044, name=zkConnectionManagerCallback-888-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=2663, name=zkConnectionManagerCallback-888-thread-1-SendThread(127.0.0.1:44283), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   2) Thread[id=2044, name=zkConnectionManagerCallback-888-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([E5AC5A05DB47924]:0)




Build Log:
[...truncated 13060 lines...]
   [junit4] Suite: org.apache.solr.security.hadoop.TestSolrCloudWithHadoopAuthPlugin
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestSolrCloudWithHadoopAuthPlugin_94F000C2E7ADD4A4-001/init-core-data-001
   [junit4]   2> 914304 WARN  (SUITE-TestSolrCloudWithHadoopAuthPlugin-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=19 numCloses=19
   [junit4]   2> 914304 INFO  (SUITE-TestSolrCloudWithHadoopAuthPlugin-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 914306 INFO  (SUITE-TestSolrCloudWithHadoopAuthPlugin-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 914306 INFO  (SUITE-TestSolrCloudWithHadoopAuthPlugin-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 919740 WARN  (SUITE-TestSolrCloudWithHadoopAuthPlugin-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.d.s.c.DefaultDirectoryService You didn't change the admin password of directory service instance 'DefaultKrbServer'.  Please update the admin password as soon as possible to prevent a possible security breach.
   [junit4]   2> 920644 INFO  (SUITE-TestSolrCloudWithHadoopAuthPlugin-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1828, maxMBSortInHeap=7.618180200441445, sim=RandomSimilarity(queryNorm=false): {}, locale=ko-KR, timezone=America/Atka
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=146832544,total=482869248
   [junit4]   2> NOTE: All tests run in this JVM: [OverriddenZkACLAndCredentialsProvidersTest, TestLockTree, SolrCloudExampleTest, TestRequestStatusCollectionAPI, ZkControllerTest, TestRecoveryHdfs, SimplePostToolTest, TestSweetSpotSimilarityFactory, CursorMarkTest, TestCollapseQParserPlugin, TestInPlaceUpdatesDistrib, LeaderFailoverAfterPartitionTest, PeerSyncTest, XsltUpdateRequestHandlerTest, TestHdfsCloudBackupRestore, TestSolrCoreParser, SegmentsInfoRequestHandlerTest, TestManagedSchema, TestScoreJoinQPNoScore, TestLegacyNumericRangeQueryBuilder, TestConfigSets, TestQueryUtils, TestPostingsSolrHighlighter, TestSolrCloudWithHadoopAuthPlugin]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithHadoopAuthPlugin -Dtests.seed=94F000C2E7ADD4A4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ko-KR -Dtests.timezone=America/Atka -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithHadoopAuthPlugin (suite) <<<
   [junit4]    > Throwable #1: java.net.BindException: Address already in use
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([94F000C2E7ADD4A4]:0)
   [junit4]    > 	at sun.nio.ch.Net.bind0(Native Method)
   [junit4]    > 	at sun.nio.ch.Net.bind(Net.java:433)
   [junit4]    > 	at sun.nio.ch.Net.bind(Net.java:425)
   [junit4]    > 	at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
   [junit4]    > 	at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
   [junit4]    > 	at org.apache.mina.transport.socket.nio.NioSocketAcceptor.open(NioSocketAcceptor.java:252)
   [junit4]    > 	at org.apache.mina.transport.socket.nio.NioSocketAcceptor.open(NioSocketAcceptor.java:49)
   [junit4]    > 	at org.apache.mina.core.polling.AbstractPollingIoAcceptor.registerHandles(AbstractPollingIoAcceptor.java:525)
   [junit4]    > 	at org.apache.mina.core.polling.AbstractPollingIoAcceptor.access$200(AbstractPollingIoAcceptor.java:67)
   [junit4]    > 	at org.apache.mina.core.polling.AbstractPollingIoAcceptor$Acceptor.run(AbstractPollingIoAcceptor.java:409)
   [junit4]    > 	at org.apache.mina.util.NamePreservingRunnable.run(NamePreservingRunnable.java:65)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [82/794 (1!)] on J1 in 6.35s, 0 tests, 1 error <<< FAILURES!

[...truncated 203 lines...]
   [junit4] Suite: org.apache.solr.TestDistributedSearch
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/init-core-data-001
   [junit4]   2> 1391787 WARN  (SUITE-TestDistributedSearch-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=64 numCloses=64
   [junit4]   2> 1391787 INFO  (SUITE-TestDistributedSearch-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1391788 INFO  (SUITE-TestDistributedSearch-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-9061)
   [junit4]   2> 1391788 INFO  (SUITE-TestDistributedSearch-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1391788 INFO  (SUITE-TestDistributedSearch-seed#[94F000C2E7ADD4A4]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /a_/t
   [junit4]   2> 1392323 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/control/cores/collection1
   [junit4]   2> 1392337 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T17:27:37-04:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1392343 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1392343 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1392343 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 1392345 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2640b9f1{/a_/t,null,AVAILABLE}
   [junit4]   2> 1392346 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3eaf0a2c{HTTP/1.1,[http/1.1]}{127.0.0.1:39618}
   [junit4]   2> 1392346 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.Server Started @1392472ms
   [junit4]   2> 1392346 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/a_/t, hostPort=39618, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/control/cores}
   [junit4]   2> 1392347 ERROR (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1392347 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1392347 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1392347 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1392347 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1392347 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-07T13:43:16.298Z
   [junit4]   2> 1392347 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/control/solr.xml
   [junit4]   2> 1392404 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1392404 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1392405 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1392579 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9
   [junit4]   2> 1392615 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9
   [junit4]   2> 1392615 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9
   [junit4]   2> 1392630 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/control/cores
   [junit4]   2> 1392630 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1392630 INFO  (coreLoadExecutor-2719-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1392631 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1392773 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 1392953 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1393409 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1393446 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from configset /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 1393446 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9
   [junit4]   2> 1393446 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1393447 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/control/cores/collection1/data/]
   [junit4]   2> 1393468 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6114943695321569]
   [junit4]   2> 1393701 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1393701 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1393702 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1393702 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1393703 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@139ec038
   [junit4]   2> 1393715 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5aa4c099[collection1] main]
   [junit4]   2> 1393716 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 1393736 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1393851 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1393852 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 1393852 INFO  (coreLoadExecutor-2719-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597095121049878528
   [junit4]   2> 1393852 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 1393852 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1393852 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1393852 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1393852 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 1393853 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 1393853 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 1393890 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 1393891 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5aa4c099[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1394458 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/shard0/cores/collection1
   [junit4]   2> 1394460 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T17:27:37-04:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1394461 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1394461 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1394461 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 1394461 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e148d0d{/a_/t,null,AVAILABLE}
   [junit4]   2> 1394461 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1365a903{HTTP/1.1,[http/1.1]}{127.0.0.1:39174}
   [junit4]   2> 1394461 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.e.j.s.Server Started @1394587ms
   [junit4]   2> 1394461 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/a_/t, hostPort=39174, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/shard0/cores}
   [junit4]   2> 1394462 ERROR (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1394462 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1394462 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1394474 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1394474 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1394474 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-07T13:43:18.425Z
   [junit4]   2> 1394475 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/shard0/solr.xml
   [junit4]   2> 1394495 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1394495 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1394496 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1394624 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9
   [junit4]   2> 1394674 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9
   [junit4]   2> 1394674 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9
   [junit4]   2> 1394676 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/shard0/cores
   [junit4]   2> 1394676 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1394684 INFO  (coreLoadExecutor-2730-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1394684 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1394822 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 1394898 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1395304 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1395340 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from configset /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 1395340 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@457bdbf9
   [junit4]   2> 1395340 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1395340 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.TestDistributedSearch_94F000C2E7ADD4A4-001/tempDir-001/shard0/cores/collection1/data/]
   [junit4]   2> 1395342 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6114943695321569]
   [junit4]   2> 1395595 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1395595 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1395596 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1395596 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1395597 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@36308290
   [junit4]   2> 1395597 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6f5a0421[collection1] main]
   [junit4]   2> 1395614 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 1395638 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1395765 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1395765 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 1395765 INFO  (coreLoadExecutor-2730-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1597095123055804416
   [junit4]   2> 1395766 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 1395766 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1395766 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1395766 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1395766 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 1395766 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 1395767 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 1395767 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 1395768 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6f5a0421[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1395785 INFO  (TEST-TestDistributedSearch.test-seed#[94F000C2E7ADD4A4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1395806 INFO  (qtp51324735-9436) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1597095123080970240)} 0 16
   [junit4]   2> 1395819 INFO  (qtp1335863626-9453) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1597095123099844608)} 0 12
   [junit4]   2> 1395830 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[1 (1597095123113476096)]} 0 9
   [junit4]   2> 1395852 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[1 (1597095123123961856)]} 0 21
   [junit4]   2> 1395886 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[2 (1597095123148079104)]} 0 33
   [junit4]   2> 1395920 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[2 (1597095123182682112)]} 0 33
   [junit4]   2> 1395921 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[3 (1597095123219382272)]} 0 0
   [junit4]   2> 1395922 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[3 (1597095123220430848)]} 0 0
   [junit4]   2> 1395944 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[4 (1597095123220430848)]} 0 21
   [junit4]   2> 1395955 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[4 (1597095123255033856)]} 0 0
   [junit4]   2> 1395956 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[5 (1597095123256082432)]} 0 0
   [junit4]   2> 1395959 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[5 (1597095123259228160)]} 0 0
   [junit4]   2> 1395976 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[6 (1597095123276005376)]} 0 0
   [junit4]   2> 1395976 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[6 (1597095123277053952)]} 0 0
   [junit4]   2> 1395977 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[7 (1597095123278102528)]} 0 0
   [junit4]   2> 1395978 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[7 (1597095123278102528)]} 0 0
   [junit4]   2> 1395978 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[8 (1597095123279151104)]} 0 0
   [junit4]   2> 1395979 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[8 (1597095123279151104)]} 0 0
   [junit4]   2> 1395988 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[9 (1597095123280199680)]} 0 8
   [junit4]   2> 1395988 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[9 (1597095123289636864)]} 0 0
   [junit4]   2> 1395988 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1597095123289636864,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1395989 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1853b74f commitCommandVersion:1597095123289636864
   [junit4]   2> 1396190 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@e70a165[collection1] main]
   [junit4]   2> 1396192 INFO  (searcherExecutor-2720-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@e70a165[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):c9)))}
   [junit4]   2> 1396192 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1396192 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 204
   [junit4]   2> 1396194 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1597095123505643520,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1396194 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@45543fe6 commitCommandVersion:1597095123505643520
   [junit4]   2> 1396291 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2b1fc6ae[collection1] main]
   [junit4]   2> 1396308 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1396309 INFO  (searcherExecutor-2731-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2b1fc6ae[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):c9)))}
   [junit4]   2> 1396309 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 115
   [junit4]   2> 1396395 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[10 (1597095123627278336)]} 0 84
   [junit4]   2> 1396396 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[10 (1597095123716407296)]} 0 1
   [junit4]   2> 1396434 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[11 (1597095123718504448)]} 0 37
   [junit4]   2> 1396491 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[11 (1597095123758350336)]} 0 55
   [junit4]   2> 1396492 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[12 (1597095123817070592)]} 0 0
   [junit4]   2> 1396492 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[12 (1597095123818119168)]} 0 0
   [junit4]   2> 1396493 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[13 (1597095123819167744)]} 0 0
   [junit4]   2> 1396493 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[13 (1597095123819167744)]} 0 0
   [junit4]   2> 1396494 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[1001 (1597095123820216320)]} 0 0
   [junit4]   2> 1396504 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[1001 (1597095123830702080)]} 0 0
   [junit4]   2> 1396505 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[14 (1597095123830702080)]} 0 0
   [junit4]   2> 1396505 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[14 (1597095123831750656)]} 0 0
   [junit4]   2> 1396506 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[15 (1597095123832799232)]} 0 0
   [junit4]   2> 1396506 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[15 (1597095123832799232)]} 0 0
   [junit4]   2> 1396507 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[16 (1597095123833847808)]} 0 0
   [junit4]   2> 1396525 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[16 (1597095123852722176)]} 0 0
   [junit4]   2> 1396540 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[17 (1597095123867402240)]} 0 0
   [junit4]   2> 1396540 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[17 (1597095123868450816)]} 0 0
   [junit4]   2> 1396541 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[100 (1597095123869499392)]} 0 0
   [junit4]   2> 1396541 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[100 (1597095123869499392)]} 0 0
   [junit4]   2> 1396542 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[101 (1597095123870547968)]} 0 0
   [junit4]   2> 1396542 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[101 (1597095123870547968)]} 0 0
   [junit4]   2> 1396543 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[102 (1597095123871596544)]} 0 0
   [junit4]   2> 1396543 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[102 (1597095123871596544)]} 0 0
   [junit4]   2> 1396561 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[103 (1597095123890470912)]} 0 0
   [junit4]   2> 1396563 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[103 (1597095123892568064)]} 0 0
   [junit4]   2> 1396564 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[104 (1597095123892568064)]} 0 0
   [junit4]   2> 1396564 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[104 (1597095123893616640)]} 0 0
   [junit4]   2> 1396565 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[105 (1597095123893616640)]} 0 0
   [junit4]   2> 1396565 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[105 (1597095123894665216)]} 0 0
   [junit4]   2> 1396566 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[106 (1597095123895713792)]} 0 0
   [junit4]   2> 1396566 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[106 (1597095123895713792)]} 0 0
   [junit4]   2> 1396567 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[107 (1597095123896762368)]} 0 0
   [junit4]   2> 1396567 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[107 (1597095123896762368)]} 0 0
   [junit4]   2> 1396568 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[108 (1597095123896762369)]} 0 0
   [junit4]   2> 1396593 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[108 (1597095123922976768)]} 0 0
   [junit4]   2> 1396593 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[109 (1597095123924025344)]} 0 0
   [junit4]   2> 1396593 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[109 (1597095123924025344)]} 0 0
   [junit4]   2> 1396594 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[110 (1597095123925073920)]} 0 0
   [junit4]   2> 1396594 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[110 (1597095123925073920)]} 0 0
   [junit4]   2> 1396595 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[111 (1597095123926122496)]} 0 0
   [junit4]   2> 1396595 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[111 (1597095123926122496)]} 0 0
   [junit4]   2> 1396596 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[112 (1597095123927171072)]} 0 0
   [junit4]   2> 1396609 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[112 (1597095123939753984)]} 0 0
   [junit4]   2> 1396609 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[113 (1597095123940802560)]} 0 0
   [junit4]   2> 1396609 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[113 (1597095123940802560)]} 0 0
   [junit4]   2> 1396610 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[114 (1597095123941851136)]} 0 0
   [junit4]   2> 1396610 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[114 (1597095123941851136)]} 0 0
   [junit4]   2> 1396611 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[115 (1597095123942899712)]} 0 0
   [junit4]   2> 1396611 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[115 (1597095123942899712)]} 0 0
   [junit4]   2> 1396612 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[116 (1597095123943948288)]} 0 0
   [junit4]   2> 1396612 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[116 (1597095123943948288)]} 0 0
   [junit4]   2> 1396632 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[117 (1597095123964919808)]} 0 0
   [junit4]   2> 1396632 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[117 (1597095123964919808)]} 0 0
   [junit4]   2> 1396633 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[118 (1597095123965968384)]} 0 0
   [junit4]   2> 1396644 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[118 (1597095123977502720)]} 0 0
   [junit4]   2> 1396653 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[119 (1597095123986939904)]} 0 0
   [junit4]   2> 1396654 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[119 (1597095123987988480)]} 0 0
   [junit4]   2> 1396654 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[120 (1597095123987988480)]} 0 0
   [junit4]   2> 1396655 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[120 (1597095123987988481)]} 0 0
   [junit4]   2> 1396655 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[121 (1597095123989037056)]} 0 0
   [junit4]   2> 1396655 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[121 (1597095123989037056)]} 0 0
   [junit4]   2> 1396656 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[122 (1597095123990085632)]} 0 0
   [junit4]   2> 1396656 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[122 (1597095123990085632)]} 0 0
   [junit4]   2> 1396669 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[123 (1597095123991134208)]} 0 0
   [junit4]   2> 1396670 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[123 (1597095124003717120)]} 0 0
   [junit4]   2> 1396670 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[124 (1597095124004765696)]} 0 0
   [junit4]   2> 1396670 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[124 (1597095124004765696)]} 0 0
   [junit4]   2> 1396671 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[125 (1597095124005814272)]} 0 0
   [junit4]   2> 1396671 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[125 (1597095124005814272)]} 0 0
   [junit4]   2> 1396672 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[126 (1597095124006862848)]} 0 0
   [junit4]   2> 1396672 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[126 (1597095124006862848)]} 0 0
   [junit4]   2> 1396673 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[127 (1597095124007911424)]} 0 0
   [junit4]   2> 1396691 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[127 (1597095124025737216)]} 0 17
   [junit4]   2> 1396691 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[128 (1597095124026785792)]} 0 0
   [junit4]   2> 1396692 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[128 (1597095124026785792)]} 0 0
   [junit4]   2> 1396692 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[129 (1597095124027834368)]} 0 0
   [junit4]   2> 1396692 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[129 (1597095124027834368)]} 0 0
   [junit4]   2> 1396693 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[130 (1597095124028882944)]} 0 0
   [junit4]   2> 1396706 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[130 (1597095124028882944)]} 0 12
   [junit4]   2> 1396706 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[131 (1597095124042514432)]} 0 0
   [junit4]   2> 1396707 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[131 (1597095124042514432)]} 0 0
   [junit4]   2> 1396707 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[132 (1597095124043563008)]} 0 0
   [junit4]   2> 1396707 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[132 (1597095124043563008)]} 0 0
   [junit4]   2> 1396708 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[133 (1597095124044611584)]} 0 0
   [junit4]   2> 1396708 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[133 (1597095124044611584)]} 0 0
   [junit4]   2> 1396709 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[134 (1597095124045660160)]} 0 0
   [junit4]   2> 1396709 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[134 (1597095124045660160)]} 0 0
   [junit4]   2> 1396710 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[135 (1597095124045660161)]} 0 0
   [junit4]   2> 1396726 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[135 (1597095124063485952)]} 0 0
   [junit4]   2> 1396727 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[136 (1597095124064534528)]} 0 0
   [junit4]   2> 1396727 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[136 (1597095124064534528)]} 0 0
   [junit4]   2> 1396728 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[137 (1597095124065583104)]} 0 0
   [junit4]   2> 1396728 INFO  (qtp1335863626-9455) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[137 (1597095124065583104)]} 0 0
   [junit4]   2> 1396729 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[138 (1597095124066631680)]} 0 0
   [junit4]   2> 1396729 INFO  (qtp1335863626-9458) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[138 (1597095124066631680)]} 0 0
   [junit4]   2> 1396730 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[139 (1597095124066631681)]} 0 0
   [junit4]   2> 1396742 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[139 (1597095124080263168)]} 0 0
   [junit4]   2> 1396743 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[140 (1597095124081311744)]} 0 0
   [junit4]   2> 1396743 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[140 (1597095124081311744)]} 0 0
   [junit4]   2> 1396744 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[141 (1597095124082360320)]} 0 0
   [junit4]   2> 1396744 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[141 (1597095124082360320)]} 0 0
   [junit4]   2> 1396745 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[142 (1597095124083408896)]} 0 0
   [junit4]   2> 1396745 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[142 (1597095124083408896)]} 0 0
   [junit4]   2> 1396746 INFO  (qtp51324735-9438) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[143 (1597095124084457472)]} 0 0
   [junit4]   2> 1396763 INFO  (qtp1335863626-9457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[143 (1597095124102283264)]} 0 0
   [junit4]   2> 1396763 INFO  (qtp51324735-9440) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/a_/t path=/update params={wt=javabin&version=2}{add=[144 (1597095124102283264)]} 0 0
   [junit4]   2> 1396764 INFO  (qtp13

[...truncated too long message...]

lity-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1083521569
     [copy] Copying 40 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1083521569
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/null1083521569

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null192745772
     [copy] Copying 246 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null192745772
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null192745772

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 41ms :: artifacts dl 2ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 439 minutes 28 seconds
Archiving artifacts
WARN: No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
WARN: java.lang.InterruptedException: no matches found within 10000
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 193 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/193/

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:545)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)  at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2069)  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
	at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
	at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)
	at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2069)
	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([5B9A4CE3EE626C9E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
	at sun.reflect.GeneratedMethodAccessor41.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest:     1) Thread[id=21843, name=OverseerAutoScalingTriggerThread-73581053365780482-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]         at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128)         at java.io.PrintStream.write(PrintStream.java:480)         at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53)         at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263)         at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:295)         at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:304)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:179)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)         at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)         at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)         at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)         at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)         at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)         at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)         at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)         at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)         at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)         at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest: 
   1) Thread[id=21843, name=OverseerAutoScalingTriggerThread-73581053365780482-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
        at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128)
        at java.io.PrintStream.write(PrintStream.java:480)
        at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53)
        at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263)
        at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:295)
        at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:304)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:179)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
        at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
        at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
        at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)
        at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
        at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)
        at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)
        at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([5B9A4CE3EE626C9E]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:47928: ADDREPLICA failed to create replica

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:47928: ADDREPLICA failed to create replica
	at __randomizedtesting.SeedInfo.seed([5B9A4CE3EE626C9E:D3CE7339409E0166]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:802)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:375)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest:     1) Thread[id=32424, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=32440, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=32431, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest: 
   1) Thread[id=32424, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=32440, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=32431, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([5B9A4CE3EE626C9E]:0)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([5B9A4CE3EE626C9E:68286427E3D5B629]:0)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:458)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:129)
	at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:113)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:292)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:372)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:113)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:78)
	at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:702)
	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
	at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
	at org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit(TestDocTermOrds.java:188)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)




Build Log:
[...truncated 13883 lines...]
   [junit4] Suite: org.apache.solr.cloud.ZkControllerTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ZkControllerTest_5B9A4CE3EE626C9E-001/init-core-data-001
   [junit4]   2> 2164446 WARN  (SUITE-ZkControllerTest-seed#[5B9A4CE3EE626C9E]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
   [junit4]   2> 2164449 INFO  (SUITE-ZkControllerTest-seed#[5B9A4CE3EE626C9E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2164451 INFO  (SUITE-ZkControllerTest-seed#[5B9A4CE3EE626C9E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 2164451 INFO  (SUITE-ZkControllerTest-seed#[5B9A4CE3EE626C9E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2164490 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testGetHostName
   [junit4]   2> 2164490 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2164499 INFO  (Thread-10209) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2164499 INFO  (Thread-10209) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2164614 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [    ] o.a.s.c.ZkTestServer start zk server on port:36183
   [junit4]   2> 2164705 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0: ZooKeeperServer not running
   [junit4]   2> 2164720 ERROR (Thread-10209) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2166070 INFO  (zkConnectionManagerCallback-3364-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2166099 INFO  (zkConnectionManagerCallback-3366-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2166132 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@33da5775, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2166159 INFO  (zkConnectionManagerCallback-3370-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2166326 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2166326 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
   [junit4]   2> 2166327 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=73581053092560898-127.0.0.1:8983_solr-n_0000000000) starting
   [junit4]   2> 2166426 INFO  (zkConnectionManagerCallback-3377-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2166428 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36183/solr ready
   [junit4]   2> 2166450 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:8983_solr
   [junit4]   2> 2166451 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=73581053092560898-127.0.0.1:8983_solr-n_0000000000) closing
   [junit4]   2> 2166452 INFO  (OverseerStateUpdate-73581053092560898-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2166470 INFO  (OverseerStateUpdate-73581053092560898-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:8983_solr
   [junit4]   2> 2166542 WARN  (OverseerExitThread) [    ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
   [junit4]   2> 2166658 WARN  (OverseerAutoScalingTriggerThread-73581053092560898-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed, exiting.
   [junit4]   2> 2166659 INFO  (zkCallback-3376-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2166692 INFO  (OverseerExitThread) [    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
   [junit4]   2> 2166694 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=765938241
   [junit4]   2> 2166694 ERROR (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.SolrCores No transient handler has been defined. Check solr.xml to see if an attempt to provide a custom TransientSolrCoreCacheFactory was done incorrectly since the default should have been used otherwise.
   [junit4]   2> 2166694 ERROR (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2166699 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36183 36183
   [junit4]   2> 2166835 INFO  (Thread-10209) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36183 36183
   [junit4]   2> 2166842 WARN  (Thread-10209) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> 2166843 INFO  (TEST-ZkControllerTest.testGetHostName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.SolrTestCaseJ4 ###Ending testGetHostName
   [junit4]   2> 2166927 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeNameUrlConversion
   [junit4]   2> 2166928 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2166935 INFO  (Thread-10210) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2166935 INFO  (Thread-10210) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2166952 ERROR (Thread-10210) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2167035 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer start zk server on port:52877
   [junit4]   2> 2167167 INFO  (zkConnectionManagerCallback-3379-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2167267 INFO  (zkConnectionManagerCallback-3381-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2167311 INFO  (zkConnectionManagerCallback-3383-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2167664 ERROR (zkCallback-3382-thread-1) [    ] o.a.s.c.c.ZkStateReader Error reading cluster properties from zookeeper
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterprops.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.loadClusterProperties(ZkStateReader.java:937) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.lambda$new$4(ZkStateReader.java:929) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:269) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_152]
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 2167664 ERROR (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2167667 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52877 52877
   [junit4]   2> 2167798 INFO  (Thread-10210) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52877 52877
   [junit4]   2> 2167802 WARN  (Thread-10210) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> 2167803 INFO  (TEST-ZkControllerTest.testNodeNameUrlConversion-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeNameUrlConversion
   [junit4]   2> 2167873 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.SolrTestCaseJ4 ###Starting testReadConfigName
   [junit4]   2> 2167874 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2167874 INFO  (Thread-10211) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2167874 INFO  (Thread-10211) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2167875 ERROR (Thread-10211) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2167974 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer start zk server on port:37651
   [junit4]   2> 2168063 INFO  (zkConnectionManagerCallback-3389-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168202 WARN  (OverseerExitThread) [    ] o.a.s.c.Overseer Unable to rejoinElection 
   [junit4]   2> org.apache.solr.common.SolrException: Unable to rejoin election
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2072) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331) ~[java/:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$2(SolrZkClient.java:304) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:304) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:483) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:438) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:425) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:839) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2069) ~[java/:?]
   [junit4]   2> 	... 2 more
   [junit4]   2> 2168207 INFO  (zkConnectionManagerCallback-3391-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168220 INFO  (zkConnectionManagerCallback-3393-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168232 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@33da5775, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2168235 INFO  (zkConnectionManagerCallback-3397-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168388 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2168389 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
   [junit4]   2> 2168390 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=73581053312434179-127.0.0.1:8983_solr-n_0000000000) starting
   [junit4]   2> 2168399 INFO  (zkConnectionManagerCallback-3404-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168402 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37651/solr ready
   [junit4]   2> 2168413 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:8983_solr
   [junit4]   2> 2168415 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=73581053312434179-127.0.0.1:8983_solr-n_0000000000) closing
   [junit4]   2> 2168416 INFO  (OverseerStateUpdate-73581053312434179-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:8983_solr
   [junit4]   2> 2168422 INFO  (zkCallback-3396-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2168422 WARN  (OverseerAutoScalingTriggerThread-73581053312434179-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException: null
   [junit4]   2> 	at java.lang.Object.wait(Native Method) ~[?:1.8.0_152]
   [junit4]   2> 	at java.lang.Object.wait(Object.java:502) ~[?:1.8.0_152]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1409) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1328) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.lambda$setData$6(SolrZkClient.java:352) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.setData(SolrZkClient.java:352) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ZkDistribStateManager.setData(ZkDistribStateManager.java:146) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:136) [java/:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> 2168425 INFO  (zkCallback-3403-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2168440 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x105699400e30004, likely client has closed socket
   [junit4]   2> 2168440 WARN  (OverseerExitThread) [    ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
   [junit4]   2> 2168442 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x105699400e30003, likely client has closed socket
   [junit4]   2> 2168442 WARN  (zkCallback-3396-thread-1) [    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2168442 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=623953033
   [junit4]   2> 2168442 ERROR (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.SolrCores No transient handler has been defined. Check solr.xml to see if an attempt to provide a custom TransientSolrCoreCacheFactory was done incorrectly since the default should have been used otherwise.
   [junit4]   2> 2168442 ERROR (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2168445 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37651 37651
   [junit4]   2> 2168585 INFO  (Thread-10211) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37651 37651
   [junit4]   2> 2168588 WARN  (Thread-10211) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> 2168593 INFO  (TEST-ZkControllerTest.testReadConfigName-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.SolrTestCaseJ4 ###Ending testReadConfigName
   [junit4]   2> 2168669 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.SolrTestCaseJ4 ###Starting testPublishAndWaitForDownStates
   [junit4]   2> 2168678 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2168685 INFO  (Thread-10212) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2168688 INFO  (Thread-10212) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2168708 ERROR (Thread-10212) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2168789 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkTestServer start zk server on port:57361
   [junit4]   2> 2168825 INFO  (zkConnectionManagerCallback-3406-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168840 INFO  (zkConnectionManagerCallback-3408-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168856 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@33da5775, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2168860 DEBUG (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$212/747992159@56ba571c
   [junit4]   2> 2168872 INFO  (zkConnectionManagerCallback-3412-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168975 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
   [junit4]   2> 2168975 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 2168975 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 2168976 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 2168976 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 2168976 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
   [junit4]   2> 2168979 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2168979 DEBUG (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/73581053365780482-127.0.0.1:8983_solr-n_0000000000
   [junit4]   2> 2168980 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:8983_solr
   [junit4]   2> 2168980 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.Overseer Overseer (id=73581053365780482-127.0.0.1:8983_solr-n_0000000000) starting
   [junit4]   2> 2168988 INFO  (zkConnectionManagerCallback-3419-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2168989 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
   [junit4]   2> 2168989 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 2168989 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
   [junit4]   2> 2168990 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 2168990 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
   [junit4]   2> 2168990 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57361/solr ready
   [junit4]   2> 2168992 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 2168993 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2168993 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2168993 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2168993 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2168993 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2168993 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2168994 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
   [junit4]   2> 2169003 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:8983_solr
   [junit4]   2> 2169004 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 2169004 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
   [junit4]   2> 2169006 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/autoscaling.json
   [junit4]   2> 2169016 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue-work
   [junit4]   2> 2169017 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 2169017 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 2169017 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
   [junit4]   2> 2169018 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 2169020 INFO  (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2169020 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 2169021 DEBUG (TEST-ZkControllerTest.testPublishAndWaitForDownStates-seed#[5B9A4CE3EE626C9E]) [n:127.0.0.1:8983_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:8983_solr as DOWN
   [junit4]   2> 2169044 INFO  (zkCallback-3418-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2169054 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
   [junit4]   2> 2169054 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 2169054 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 2169056 DEBUG (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.ClusterStateMutator building a new cName: testPublishAndWaitForDownStates
   [junit4]   2> 2169056 DEBUG (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 2169057 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 2169057 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
   [junit4]   2> 2169057 INFO  (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "core":"testPublishAndWaitForDownStates",
   [junit4]   2>   "node_name":"non_existent_host1",
   [junit4]   2>   "collection":"testPublishAndWaitForDownStates",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "operation":"addreplica"} 
   [junit4]   2> 2169058 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.scheduled_maintenance
   [junit4]   2> 2169059 DEBUG (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
   [junit4]   2>   "range":"80000000-7fffffff",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "replicas":{}}
   [junit4]   2> 2169059 DEBUG (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator New Slice: shard1:{
   [junit4]   2>   "range":"80000000-7fffffff",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "replicas":{"core_node21":{
   [junit4]   2>       "core":"testPublishAndWaitForDownStates",
   [junit4]   2>       "base_url":null,
   [junit4]   2>       "state":"active",
   [junit4]   2>       "node_name":"non_existent_host1",
   [junit4]   2>       "type":null}}}
   [junit4]   2> 2169059 INFO  (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "core":"testPublishAndWaitForDownStates",
   [junit4]   2>   "node_name":"non_existent_host2",
   [junit4]   2>   "collection":"testPublishAndWaitForDownStates",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "operation":"addreplica"} 
   [junit4]   2> 2169061 DEBUG (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
   [junit4]   2>   "range":"80000000-7fffffff",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "replicas":{"core_node21":{
   [junit4]   2>       "core":"testPublishAndWaitForDownStates",
   [junit4]   2>       "base_url":null,
   [junit4]   2>       "state":"active",
   [junit4]   2>       "node_name":"non_existent_host1",
   [junit4]   2>       "type":null}}}
   [junit4]   2> 2169061 DEBUG (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator New Slice: shard1:{
   [junit4]   2>   "range":"80000000-7fffffff",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "replicas":{
   [junit4]   2>     "core_node21":{
   [junit4]   2>       "core":"testPublishAndWaitForDownStates",
   [junit4]   2>       "base_url":null,
   [junit4]   2>       "state":"active",
   [junit4]   2>       "node_name":"non_existent_host1",
   [junit4]   2>       "type":null},
   [junit4]   2>     "core_node22":{
   [junit4]   2>       "core":"testPublishAndWaitForDownStates",
   [junit4]   2>       "base_url":null,
   [junit4]   2>       "state":"down",
   [junit4]   2>       "node_name":"non_existent_host2",
   [junit4]   2>       "type":null}}}
   [junit4]   2> 2169161 DEBUG (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:8983_solr
   [junit4]   2> 2169162 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 2169162 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 2169164 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 2169263 DEBUG (OverseerStateUpdate-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/testPublishAndWaitForDownStates/state.json version: 0
   [junit4]   2> 2169263 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: data: /solr/collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 2169264 INFO  (zkCallback-3411-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testPublishAndWaitForDownStates/state.json] for collection [testPublishAndWaitForDownStates] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2169264 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: data: /solr/collections/testPublishAndWaitForDownStates/state.json
   [junit4]   2> 2169264 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
   [junit4]   2> 2169265 DEBUG (SyncThread:0) [    ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
   [junit4]   2> 2169941 WARN  (OverseerExitThread) [    ] o.a.s.c.Overseer Unable to rejoinElection 
   [junit4]   2> org.apache.solr.common.SolrException: Unable to rejoin election
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2072) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331) ~[java/:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152]
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1535) ~[zookeeper-3.4.11.jar:3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.lambda$getChildren$4(SolrZkClient.java:328) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393) ~[java/:?]
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2069) ~[java/:?]
   [junit4]   2> 	... 2 more
   [junit4]   2> 2170994 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2170995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2170995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2170995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2170995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2170995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2170995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2172995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2172995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2172995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2172995 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2172996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2172996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2172996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2174996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2174996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2174996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2174996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2174996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2174996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2174996 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2176997 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2176998 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2176998 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2176998 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2176998 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2176998 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2176998 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2178998 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2178999 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2178999 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2178999 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2178999 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2178999 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2178999 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2180999 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2180999 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2181000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2181000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2181000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2181000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2181000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2183000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2183000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2183000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2183000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2183000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2183000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2183000 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2185001 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2185001 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2185001 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2185001 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2185001 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2185001 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2185001 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2187001 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2187002 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2187002 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2187002 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2187002 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2187002 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2187002 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2189002 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2189003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2189003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2189003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2189003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2189003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2189003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2191003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2191003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2191003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2191003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2191003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2191003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2191003 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2193005 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2193006 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2193006 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2193006 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2193006 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2193006 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2193006 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2195006 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2195007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2195007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2195007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2195007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2195007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2195007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2197007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2197007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2197007 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2197008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2197008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2197008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2197008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2199008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2199008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2199008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2199008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2199008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2199008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2199008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2201008 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2201009 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2201009 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2201009 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2201009 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2201009 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2201009 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2203009 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2203010 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2203010 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2203010 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2203010 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2203010 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2203010 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2205010 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2205011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2205011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2205011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2205011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2205011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2205011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2207011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2207011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2207011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2207011 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2207012 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2207012 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2207012 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2209012 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2209013 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2209013 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2209013 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2209013 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2209013 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2209013 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2211013 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2211014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2211014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2211014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2211014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2211014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2211014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2213014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2213014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2213014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2213014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2213014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2213014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2213014 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2215015 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2215015 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
   [junit4]   2> 2215015 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
   [junit4]   2> 2215015 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
   [junit4]   2> 2215015 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
   [junit4]   2> 2215015 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
   [junit4]   2> 2215015 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
   [junit4]   2> 2217015 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
   [junit4]   2> 2217016 DEBUG (OverseerCollectionConfigSetProcessor-73581053365780482-127.0.0.1:8983_solr-n_

[...truncated too long message...]

SegmentReader.<init>(SegmentReader.java:78)
   [junit4]    > 	at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:702)
   [junit4]    > 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:300)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:865)
   [junit4]    > 	at org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit(TestDocTermOrds.java:188)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {field=PostingsFormat(name=Direct), foo=PostingsFormat(name=Direct), id=PostingsFormat(name=Asserting)}, docValues:{}, maxPointsInLeafNode=554, maxMBSortInHeap=7.066167841094184, sim=RandomSimilarity(queryNorm=true): {field=DFR I(ne)LZ(0.3)}, locale=en-CA, timezone=Mexico/BajaSur
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=327265640,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [InfoHandlerTest, TestDistribDocBasedVersion, ClassificationUpdateProcessorIntegrationTest, MetricTriggerIntegrationTest, TestCSVResponseWriter, AutoAddReplicasPlanActionTest, SimpleFacetsTest, TestReplicationHandlerBackup, TestIndexSearcher, PreAnalyzedUpdateProcessorTest, HdfsRecoverLeaseTest, TestCollectionsAPIViaSolrCloudCluster, TestHdfsUpdateLog, TestLeaderInitiatedRecoveryThread, TestRuleBasedAuthorizationPlugin, TestSort, SynonymTokenizerTest, SolrMetricReporterTest, ConfigSetsAPITest, TestSolrCloudSnapshots, ExitableDirectoryReaderTest, TestFilteredDocIdSet, PluginInfoTest, TestLRUStatsCache, TestXmlQParser, TestAddFieldRealTimeGet, UpdateParamsTest, TestConfigReload, MetricsConfigTest, TestSegmentSorting, CreateRoutedAliasTest, TestNestedDocsSort, TestStressLiveNodes, TestSchemaResource, TestRebalanceLeaders, ImplicitSnitchTest, TestSimpleTextCodec, SecurityConfHandlerTest, DistributedFacetPivotLongTailTest, MissingSegmentRecoveryTest, TestApiFramework, MetricUtilsTest, NodeAddedTriggerTest, DefaultValueUpdateProcessorTest, TestNRTOpen, NodeMarkersRegistrationTest, DistributedExpandComponentTest, TestMissingGroups, TestHashPartitioner, HighlighterConfigTest, ChaosMonkeyNothingIsSafeTest, TestLegacyFieldReuse, TestFieldTypeCollectionResource, SolrRequestParserTest, TestSubQueryTransformer, TestSkipOverseerOperations, TestIBSimilarityFactory, TestCoreAdminApis, TestDocSet, FieldAnalysisRequestHandlerTest, BooleanFieldTest, SolrCoreCheckLockOnStartupTest, FullSolrCloudDistribCmdsTest, TestOverriddenPrefixQueryForCustomFieldType, TestPullReplicaErrorHandling, TestClassicSimilarityFactory, AliasIntegrationTest, TestWriterPerf, V2StandaloneTest, PKIAuthenticationIntegrationTest, TestQuerySenderNoQuery, HdfsSyncSliceTest, TestOmitPositions, TestFieldCache, CdcrBidirectionalTest, TestClusterStateProvider, OverseerModifyCollectionTest, SolrIndexSplitterTest, TestManagedResource, EchoParamsTest, TestConfigSetProperties, TestIndexingPerformance, TestBackupRepositoryFactory, TestCloudPseudoReturnFields, HdfsUnloadDistributedZkTest, UUIDUpdateProcessorFallbackTest, TestFileDictionaryLookup, TestManagedSynonymFilterFactory, TestSolrCLIRunExample, DebugComponentTest, RequestHandlersTest, ZkShardTermsTest, IndexBasedSpellCheckerTest, DeleteNodeTest, TestPHPSerializedResponseWriter, TestHalfAndHalfDocValues, TestGeoJSONResponseWriter, TestUseDocValuesAsStored2, TestTrackingShardHandlerFactory, TestIntervalFaceting, TestJoin, TestDistribIDF, WrapperMergePolicyFactoryTest, CdcrUpdateLogTest, DocumentAnalysisRequestHandlerTest, TestExclusionRuleCollectionAccess, BinaryUpdateRequestHandlerTest, TestEmbeddedSolrServerConstructors, TestMaxTokenLenTokenizer, CleanupOldIndexTest, OpenExchangeRatesOrgProviderTest, BlockCacheTest, TestLuceneMatchVersion, TestReplicaProperties, TestClusterProperties, TestLegacyFieldCache, TestExpandComponent, ConcurrentDeleteAndCreateCollectionTest, HdfsChaosMonkeyNothingIsSafeTest, OutputWriterTest, TestNamedUpdateProcessors, TriLevelCompositeIdRoutingTest, TestOnReconnectListenerSupport, TestBinaryResponseWriter, RegexBytesRefFilterTest, SpellingQueryConverterTest, TestManagedSchemaThreadSafety, ZkNodePropsTest, TestMinMaxOnMultiValuedField, CdcrReplicationHandlerTest, TestFieldCacheSortRandom, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, ShardRoutingTest, ShardRoutingCustomTest, TestDistributedGrouping, TestRecovery, TestStressReorder, TestStressVersions, DistributedTermsComponentTest, StatsComponentTest, SolrCmdDistributorTest, TestFiltering, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, DistributedQueryElevationComponentTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, SpellCheckCollatorTest, SuggesterFSTTest, SuggesterTest, TestStressLucene, TestTrie, SuggesterWFSTTest, TestCSVLoader, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, TestUpdate, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, QueryEqualityTest, SortByFunctionTest, TestRemoteStreaming, DistanceFunctionTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, TestSurroundQueryParser, LukeRequestHandlerTest, TestQueryUtils, DirectSolrSpellCheckerTest, TestQueryTypes, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, TermsComponentTest, XmlUpdateRequestHandlerTest, RequiredFieldsTest, TestSolrQueryParser, FastVectorHighlighterTest, LoggingHandlerTest, SolrPluginUtilsTest, IndexSchemaTest, TestCollationField, UpdateRequestProcessorFactoryTest, MBeansHandlerTest, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, TestComponentsName, TestSolrIndexConfig, ResponseLogComponentTest, SolrIndexConfigTest, TestMergePolicyConfig, MultiTermTest, TestBinaryField, TestSearchPerf, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, SpellPossibilityIteratorTest, TestCharFilters, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestFastWriter, TimeZoneUtilsTest, TestFastOutputStream, TestFastLRUCache, PrimUtilsTest, TestSuggestSpellingConverter, ClusterStateTest, TestUtils, SliceStateTest, TestRTGBase, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ConnectionReuseTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, HttpPartitionTest, LIROnShardRestartTest, RemoteQueryErrorTest, SharedFSAutoReplicaFailoverTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudConsistency, TestCloudInspectUtil, TestCloudSearcherWarming, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestMiniSolrCloudClusterSSL, TestSSLRandomization, TestSizeLimitedDistributedMap, TestSolrCloudWithSecureImpersonation, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TestUtilizeNode, VMParamsZkACLAndCredentialsProvidersTest, ZkFailoverTest, CustomCollectionTest, TestHdfsCloudBackupRestore, AutoAddReplicasIntegrationTest, ExecutePlanActionTest, SystemLogListenerTest, TriggerSetPropertiesIntegrationTest, CdcrReplicationDistributedZkTest, CdcrVersionReplicationTest, HDFSCollectionsAPITest, RuleEngineTest, RulesTest, ConfigureRecoveryStrategyTest, CoreSorterTest, TestCorePropertiesReload, TestCustomStream, ResponseBuilderTest, SearchHandlerTest, TestHdfsBackupRestoreCore, TestSolrConfigHandlerConcurrent, TestSystemCollAutoCreate, DistributedFacetPivotWhiteBoxTest, ReplicaListTransformerTest, ShufflingReplicaListTransformerTest, SpatialHeatmapFacetsTest, SolrSlf4jReporterTest, TestGraphMLResponseWriter, TestRawTransformer, TestDynamicFieldCollectionResource, TestFieldTypeResource, TestSerializedLuceneMatchVersion, CurrencyFieldTypeTest, DateRangeFieldTest, EnumFieldTest, ManagedSchemaRoundRobinCloudTest, TestCollationFieldDocValues, TestManagedSchemaAPI, TestPointFields, AnalyticsQueryTest, CurrencyRangeFacetCloudTest, RankQueryTest, TestComplexPhraseQParserPlugin, TestCustomSort, TestHashQParserPlugin, TestNoOpRegenerator, TestPayloadScoreQParserPlugin, TestReRankQParserPlugin, TestSearcherReuse, TestSmileRequest, TestSolr4Spatial2, TestSolrCoreParser, TestSolrFieldCacheBean, TestStandardQParsers, TestStressUserVersions, TestXmlQParserPlugin, TestJsonFacetRefinement, TestJsonFacets, BlockJoinFacetSimpleTest, TestCloudNestedDocsSort, TestJsonRequest, TestExactStatsCache, TestPKIAuthenticationPlugin, TestDelegationWithHadoopAuth, TestZkAclsWithHadoopAuth, HttpSolrCallGetCoreTest, BlockDirectoryTest, TestDocTermOrds]
   [junit4] Completed [793/794 (3!)] on J0 in 179.32s, 10 tests, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20180406_052114_6868429182722461103422.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid29767.hprof ...
   [junit4] Heap dump file created [474638649 bytes in 1.669 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 9288 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid29767.hprof

Total time: 249 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 192 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/192/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
ObjectTracker found 5 object(s) that were not released!!! [SolrCore, InternalHttpClient, MockDirectoryWrapper, MockDirectoryWrapper, MockDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1040)  at org.apache.solr.core.SolrCore.reload(SolrCore.java:657)  at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1302)  at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:944)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:298)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:248)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:290)  at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1190)  at org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:696)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)  at org.apache.solr.core.SolrCore.reload(SolrCore.java:657)  at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1302)  at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:944)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:526)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)  at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:730)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1047)  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:643)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:526)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)  at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)  at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [SolrCore, InternalHttpClient, MockDirectoryWrapper, MockDirectoryWrapper, MockDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1040)
	at org.apache.solr.core.SolrCore.reload(SolrCore.java:657)
	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1302)
	at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:944)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:298)
	at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:248)
	at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:290)
	at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1190)
	at org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:696)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
	at org.apache.solr.core.SolrCore.reload(SolrCore.java:657)
	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1302)
	at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:944)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:526)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
	at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:730)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1047)
	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:643)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:526)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
	at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([B3CB0D2BE889AEA6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
	at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14338 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/init-core-data-001
   [junit4]   2> 2065841 WARN  (SUITE-TestReplicationHandler-seed#[B3CB0D2BE889AEA6]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=469 numCloses=469
   [junit4]   2> 2065841 INFO  (SUITE-TestReplicationHandler-seed#[B3CB0D2BE889AEA6]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2065842 INFO  (SUITE-TestReplicationHandler-seed#[B3CB0D2BE889AEA6]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 2065842 INFO  (SUITE-TestReplicationHandler-seed#[B3CB0D2BE889AEA6]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2065854 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testEmptyCommits
   [junit4]   2> 2065855 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-001/collection1
   [junit4]   2> 2065858 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T13:27:37-08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2065906 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2065906 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2065906 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 2065907 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@688d85cd{/solr,null,AVAILABLE}
   [junit4]   2> 2065907 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3c66203c{HTTP/1.1,[http/1.1]}{127.0.0.1:34436}
   [junit4]   2> 2065907 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.Server Started @2065961ms
   [junit4]   2> 2065907 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=34436}
   [junit4]   2> 2065908 ERROR (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2065908 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2065908 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 2065908 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 2065908 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2065908 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-05T07:50:20.660Z
   [junit4]   2> 2065908 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-001/solr.xml
   [junit4]   2> 2065912 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2065912 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2065913 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2066225 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13
   [junit4]   2> 2066258 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13
   [junit4]   2> 2066258 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13
   [junit4]   2> 2066267 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-001/.
   [junit4]   2> 2066267 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2066271 INFO  (coreLoadExecutor-6128-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2066287 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 2066308 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2066316 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 2066316 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-001/./collection1, trusted=true
   [junit4]   2> 2066316 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13
   [junit4]   2> 2066316 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2066316 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-001/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 2066320 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7431806441911083]
   [junit4]   2> 2066406 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2066406 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2066406 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=49.0283203125, floorSegmentMB=2.0673828125, forceMergeDeletesPctAllowed=21.932180843130684, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2066407 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@114ec550[collection1] main]
   [junit4]   2> 2066407 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-001/collection1/conf
   [junit4]   2> 2066408 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 2066408 INFO  (coreLoadExecutor-6128-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2066416 INFO  (searcherExecutor-6129-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@114ec550[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2066416 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-002/collection1
   [junit4]   2> 2066418 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T13:27:37-08:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 2066419 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2066419 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2066419 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 2066419 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2df1656f{/solr,null,AVAILABLE}
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4b2e131b{HTTP/1.1,[http/1.1]}{127.0.0.1:38835}
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.Server Started @2066473ms
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=38835}
   [junit4]   2> 2066420 ERROR (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-04-05T07:50:21.172Z
   [junit4]   2> 2066420 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-002/solr.xml
   [junit4]   2> 2066424 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 2066424 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 2066425 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2066508 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13
   [junit4]   2> 2066533 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13
   [junit4]   2> 2066533 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13
   [junit4]   2> 2066546 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-002/.
   [junit4]   2> 2066546 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2066562 INFO  (coreLoadExecutor-6138-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 2066574 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 2066588 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2066590 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 2066590 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-002/./collection1, trusted=true
   [junit4]   2> 2066592 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76291b13
   [junit4]   2> 2066592 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2066592 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-002/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 2066598 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7431806441911083]
   [junit4]   2> 2066877 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2066877 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2066878 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=49.0283203125, floorSegmentMB=2.0673828125, forceMergeDeletesPctAllowed=21.932180843130684, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2066879 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@72dc346[collection1] main]
   [junit4]   2> 2066879 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_B3CB0D2BE889AEA6-001/solr-instance-002/collection1/conf
   [junit4]   2> 2066886 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
   [junit4]   2> 2066886 INFO  (coreLoadExecutor-6138-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2066888 INFO  (searcherExecutor-6139-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@72dc346[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2066903 INFO  (qtp268002353-20939) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2066903 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1]} 0 0
   [junit4]   2> 2066904 INFO  (qtp268002353-20939) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2066904 INFO  (qtp268002353-20939) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@76963d4f commitCommandVersion:0
   [junit4]   2> 2066907 INFO  (qtp268002353-20939) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@b872230[collection1] main]
   [junit4]   2> 2066907 INFO  (qtp268002353-20939) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2066908 INFO  (searcherExecutor-6129-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@b872230[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1)))}
   [junit4]   2> 2066908 INFO  (qtp268002353-20939) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={commit=true&wt=javabin&version=2}{commit=} 0 4
   [junit4]   2> 2066919 WARN  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 2066922 INFO  (qtp268002353-20939) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2066926 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 2066926 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's version: 1522914621656
   [junit4]   2> 2066926 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 2066926 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 2066926 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 2066928 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&tlogFiles=false&wt=javabin&version=2&command=filelist} status=0 QTime=1
   [junit4]   2> 2066929 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Number of files in latest index in master: 10
   [junit4]   2> 2066945 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=63.8134765625, floorSegmentMB=0.9326171875, forceMergeDeletesPctAllowed=28.04361265789452, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 2066946 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2066946 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@3a7a1b31 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@30f663fa; maxCacheMB=0.1611328125 maxMergeSizeMB=0.7431640625))
   [junit4]   2> 2066947 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066954 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066957 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066958 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_BlockTreeOrds_0.tio&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066958 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_BlockTreeOrds_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066959 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066960 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066960 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066961 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_BlockTreeOrds_0.tipo&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066962 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2066962 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Bytes downloaded: 1746, Bytes skipped downloading: 0
   [junit4]   2> 2066962 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=1746) : 0 secs (null bytes/sec) to MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@3a7a1b31 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@30f663fa; maxCacheMB=0.1611328125 maxMergeSizeMB=0.7431640625))
   [junit4]   2> 2066966 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2066967 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2066967 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@25802ce7[collection1] main]
   [junit4]   2> 2066968 INFO  (searcherExecutor-6139-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@25802ce7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1)))}
   [junit4]   2> 2066968 INFO  (qtp205604491-20956) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:34436/solr/collection1/replication&command=fetchindex} status=0 QTime=49
   [junit4]   2> 2066981 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2066981 INFO  (indexFetcher-6143-thread-1) [    ] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 2066981 INFO  (indexFetcher-6143-thread-1) [    ] o.a.s.h.IndexFetcher Master's version: 1522914621656
   [junit4]   2> 2066981 INFO  (indexFetcher-6143-thread-1) [    ] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 2066981 INFO  (indexFetcher-6143-thread-1) [    ] o.a.s.h.IndexFetcher Slave's version: 1522914621656
   [junit4]   2> 2066981 INFO  (indexFetcher-6143-thread-1) [    ] o.a.s.h.IndexFetcher Slave in sync with master.
   [junit4]   2> 2067013 INFO  (qtp205604491-20960) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=name:empty1&sort=id+desc&wt=javabin&version=2} hits=1 status=0 QTime=0
   [junit4]   2> 2067013 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 1 docs
   [junit4]   2> 2067014 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2067014 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2067015 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={slave=false&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2067015 INFO  (qtp205604491-20955) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2067015 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=assertVersions&qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2067016 INFO  (qtp205604491-20961) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=assertVersions&qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2067016 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2067016 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2067016 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2067016 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={commit=true&wt=javabin&version=2}{commit=} 0 0
   [junit4]   2> 2067022 WARN  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 2067023 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2067023 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 2067023 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's version: 1522914621656
   [junit4]   2> 2067023 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 2067023 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's version: 1522914621656
   [junit4]   2> 2067023 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave in sync with master.
   [junit4]   2> 2067023 INFO  (qtp205604491-20958) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:34436/solr/collection1/replication&command=fetchindex} status=0 QTime=6
   [junit4]   2> 2067024 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 0
   [junit4]   2> 2067024 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2067024 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@76963d4f commitCommandVersion:0
   [junit4]   2> 2067029 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7d8ab30f[collection1] main]
   [junit4]   2> 2067030 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2067031 INFO  (searcherExecutor-6129-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7d8ab30f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1) Uninverting(_1(7.4.0):C1)))}
   [junit4]   2> 2067031 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={commit=true&wt=javabin&version=2}{commit=} 0 6
   [junit4]   2> 2067042 WARN  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 2067043 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2067043 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's generation: 3
   [junit4]   2> 2067043 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's version: 1522914621776
   [junit4]   2> 2067043 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 2067043 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's version: 1522914621656
   [junit4]   2> 2067043 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 2067044 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&tlogFiles=false&wt=javabin&version=2&command=filelist} status=0 QTime=0
   [junit4]   2> 2067044 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Number of files in latest index in master: 19
   [junit4]   2> 2067045 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=63.8134765625, floorSegmentMB=0.9326171875, forceMergeDeletesPctAllowed=28.04361265789452, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 2067050 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@24e56885 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@18224393; maxCacheMB=0.1611328125 maxMergeSizeMB=0.7431640625))
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.si because it already exists
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.nvd because it already exists
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fdx because it already exists
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.tio because it already exists
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.doc because it already exists
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fdt because it already exists
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fnm because it already exists
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.nvm because it already exists
   [junit4]   2> 2067051 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.tipo because it already exists
   [junit4]   2> 2067051 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1_BlockTreeOrds_0.tipo&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067089 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067090 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067090 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067091 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1_BlockTreeOrds_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067092 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067092 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067093 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1_BlockTreeOrds_0.tio&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067095 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067096 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=segments_3&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2067096 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Bytes downloaded: 1808, Bytes skipped downloading: 0
   [junit4]   2> 2067096 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=1808) : 0 secs (null bytes/sec) to MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@24e56885 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@18224393; maxCacheMB=0.1611328125 maxMergeSizeMB=0.7431640625))
   [junit4]   2> 2067136 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2067137 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2067137 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3acbe5df[collection1] main]
   [junit4]   2> 2067138 INFO  (searcherExecutor-6139-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3acbe5df[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1) Uninverting(_1(7.4.0):C1)))}
   [junit4]   2> 2067139 INFO  (qtp205604491-20961) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:34436/solr/collection1/replication&command=fetchindex} status=0 QTime=107
   [junit4]   2> 2067140 INFO  (qtp205604491-20955) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=name:empty2&sort=id+desc&wt=javabin&version=2} hits=1 status=0 QTime=0
   [junit4]   2> 2067140 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 1 docs
   [junit4]   2> 2067140 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2067140 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2067141 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={slave=false&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2067141 INFO  (qtp205604491-20961) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2067142 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=assertVersions&qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2067142 INFO  (qtp205604491-20955) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=assertVersions&qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2067144 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[3]} 0 2
   [junit4]   2> 2067144 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2067145 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@76963d4f commitCommandVersion:0
   [junit4]   2> 2067146 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3931d2f8[collection1] realtime]
   [junit4]   2> 2067146 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2067146 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={openSearcher=false&commit=true&wt=javabin&version=2}{commit=} 0 1
   [junit4]   2> 2067161 WARN  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 2067162 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2067167 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's generation: 4
   [junit4]   2> 2067167 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's version: 1522914621897
   [junit4]   2> 2067167 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's generation: 3
   [junit4]   2> 2067167 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's version: 1522914621776
   [junit4]   2> 2067168 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 2067169 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&tlogFiles=false&wt=javabin&version=2&command=filelist} status=0 QTime=0
   [junit4]   2> 2067169 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Number of files in latest index in master: 28
   [junit4]   2> 2067170 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=63.8134765625, floorSegmentMB=0.9326171875, forceMergeDeletesPctAllowed=28.04361265789452, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 2067173 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2067174 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2067174 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2068174 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2068174 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2069175 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2069175 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2070175 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2070175 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2071176 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2071176 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2072176 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2072176 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2073177 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2073177 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2074177 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2074177 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2075178 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2075178 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2076178 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2076178 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2077179 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Found unused file: segments_2
   [junit4]   2> 2077179 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Sleeping for 1000ms to wait for unused lucene index files to be delete-able
   [junit4]   2> 2078179 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher IndexFetcher slept for 11000ms for unused lucene index files to be delete-able
   [junit4]   2> 2078179 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@6e65f2e5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@29dfa632; maxCacheMB=0.1611328125 maxMergeSizeMB=0.7431640625))
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.si because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.nvd because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fdx because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.tio because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.doc because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fdt because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fnm because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.nvm because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.tipo because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1_BlockTreeOrds_0.tipo because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.si because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.nvd because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.fdx because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1_BlockTreeOrds_0.doc because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.fnm because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.nvm because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1_BlockTreeOrds_0.tio because it already exists
   [junit4]   2> 2078180 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.fdt because it already exists
   [junit4]   2> 2078182 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078190 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078191 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2_BlockTreeOrds_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078191 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2_BlockTreeOrds_0.tio&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078192 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078198 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078199 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078200 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2_BlockTreeOrds_0.tipo&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078200 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078201 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=4&qt=/replication&file=segments_4&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078201 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Bytes downloaded: 1870, Bytes skipped downloading: 0
   [junit4]   2> 2078201 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=1870) : 11 secs (170 bytes/sec) to MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@6e65f2e5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@29dfa632; maxCacheMB=0.1611328125 maxMergeSizeMB=0.7431640625))
   [junit4]   2> 2078203 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2078203 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2078204 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@c6774a2[collection1] main]
   [junit4]   2> 2078205 INFO  (searcherExecutor-6139-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@c6774a2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1) Uninverting(_1(7.4.0):C1) Uninverting(_2(7.4.0):C1)))}
   [junit4]   2> 2078205 INFO  (qtp205604491-20958) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:34436/solr/collection1/replication&command=fetchindex} status=0 QTime=11058
   [junit4]   2> 2078207 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=name:empty3&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2078207 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 0 docs
   [junit4]   2> 2078209 INFO  (qtp205604491-20961) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=name:empty3&sort=id+desc&wt=javabin&version=2} hits=1 status=0 QTime=0
   [junit4]   2> 2078209 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 1 docs
   [junit4]   2> 2078209 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[4]} 0 0
   [junit4]   2> 2078210 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2078210 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@76963d4f commitCommandVersion:0
   [junit4]   2> 2078211 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7f4afe85[collection1] main]
   [junit4]   2> 2078211 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2078212 INFO  (searcherExecutor-6129-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7f4afe85[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1) Uninverting(_1(7.4.0):C1) Uninverting(_2(7.4.0):C1) Uninverting(_3(7.4.0):C1)))}
   [junit4]   2> 2078212 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={commit=true&wt=javabin&version=2}{commit=} 0 2
   [junit4]   2> 2078213 WARN  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 2078214 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2078214 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's generation: 5
   [junit4]   2> 2078214 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Master's version: 1522914632962
   [junit4]   2> 2078214 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's generation: 4
   [junit4]   2> 2078214 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Slave's version: 1522914621897
   [junit4]   2> 2078214 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 2078265 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&tlogFiles=false&wt=javabin&version=2&command=filelist} status=0 QTime=51
   [junit4]   2> 2078266 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Number of files in latest index in master: 37
   [junit4]   2> 2078267 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=63.8134765625, floorSegmentMB=0.9326171875, forceMergeDeletesPctAllowed=28.04361265789452, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 2078267 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@4316712c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@41e8b660; maxCacheMB=0.1611328125 maxMergeSizeMB=0.7431640625))
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.si because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.nvd because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fdx because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.tio because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.doc because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fdt because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.fnm because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0.nvm because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _0_BlockTreeOrds_0.tipo because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1_BlockTreeOrds_0.tipo because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.si because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.nvd because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.fdx because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1_BlockTreeOrds_0.doc because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.fnm because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.nvm because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1_BlockTreeOrds_0.tio because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _1.fdt because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2.nvd because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2.si because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2_BlockTreeOrds_0.doc because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2_BlockTreeOrds_0.tio because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2.fdt because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2.fnm because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2.nvm because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2_BlockTreeOrds_0.tipo because it already exists
   [junit4]   2> 2078268 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Skipping download for _2.fdx because it already exists
   [junit4]   2> 2078269 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078270 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3_BlockTreeOrds_0.tipo&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078271 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078272 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078273 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078274 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078275 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078276 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3_BlockTreeOrds_0.tio&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078276 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=_3_BlockTreeOrds_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078282 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=5&qt=/replication&file=segments_5&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 2078283 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Bytes downloaded: 1932, Bytes skipped downloading: 0
   [junit4]   2> 2078283 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=1932) : 0 secs (null bytes/sec) to MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@4316712c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@41e8b660; maxCacheMB=0.1611328125 maxMergeSizeMB=0.7431640625))
   [junit4]   2> 2078286 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2078287 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2078341 INFO  (explicit-fetchindex-cmd) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@7effe3ca[collection1] main]
   [junit4]   2> 2078342 INFO  (searcherExecutor-6139-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7effe3ca[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.4.0):C1) Uninverting(_1(7.4.0):C1) Uninverting(_2(7.4.0):C1) Uninverting(_3(7.4.0):C1)))}
   [junit4]   2> 2078345 INFO  (qtp205604491-20955) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:34436/solr/collection1/replication&command=fetchindex} status=0 QTime=132
   [junit4]   2> 2078347 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=name:(empty1+empty2+empty3+empty4)&sort=id+desc&wt=javabin&version=2} hits=4 status=0 QTime=0
   [junit4]   2> 2078347 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 4 docs
   [junit4]   2> 2078348 INFO  (qtp205604491-20958) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=name:(empty1+empty2+empty3+empty4)&sort=id+desc&wt=javabin&version=2} hits=4 status=0 QTime=0
   [junit4]   2> 2078348 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 4 docs
   [junit4]   2> 2078348 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2078349 INFO  (qtp268002353-20941) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2078350 INFO  (qtp268002353-20945) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={slave=false&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2078350 INFO  (qtp205604491-20955) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
   [junit4]   2> 2078350 INFO  (qtp268002353-20944) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=assertVersions&qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2078350 INFO  (qtp205604491-20958) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=assertVersions&qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2078351 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testEmptyCommits
   [junit4]   2> 2078351 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@3c66203c{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2078351 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=876103551
   [junit4]   2> 2078352 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2078352 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3f10ade1: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@14f2c5b2
   [junit4]   2> 2078356 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2078356 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@247f4d82: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@70aff4d3
   [junit4]   2> 2078359 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2078359 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@24a49b47: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@67118203
   [junit4]   2> 2078410 INFO  (coreCloseExecutor-6148-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@57ac64ba
   [junit4]   2> 2078410 INFO  (coreCloseExecutor-6148-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=57ac64ba
   [junit4]   2> 2078410 INFO  (coreCloseExecutor-6148-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1370b10d: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@1510c53c
   [junit4]   2> 2078416 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@688d85cd{/solr,null,UNAVAILABLE}
   [junit4]   2> 2078416 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 2078422 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@4b2e131b{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2078423 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1476992021
   [junit4]   2> 2078423 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2078423 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1bcd1e23: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@1d67ccd5
   [junit4]   2> 2078428 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[B3CB0D2BE889AEA6]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2078428 INFO  (TEST

[...truncated too long message...]

est, SOLR749Test, AutoScalingHandlerTest, TestOrdValues, TestCloudRecovery, PeerSyncReplicationTest, TestTriggerIntegration, LIROnShardRestartTest, TestLRUCache, SaslZkACLProviderTest, TestLuceneIndexBackCompat, TestFieldCacheVsDocValues, TestSortableTextField, CdcrBootstrapTest, RestoreTriggerStateTest, ShardRoutingCustomTest, TestHdfsCloudBackupRestore, OverseerTest, HdfsLockFactoryTest, AssignTest, TestExactSharedStatsCache, TestSolr4Spatial, DirectSolrConnectionTest, DistanceUnitsTest, TestDocBasedVersionConstraints, JavabinLoaderTest, TestBM25SimilarityFactory, TestSolrConfigHandler, AutoscalingHistoryHandlerTest, TestZkChroot, TestRawResponseWriter, TestCustomSort, TestSolrJ, QueryEqualityTest, CoreAdminRequestStatusTest, CoreSorterTest, TestXmlQParserPlugin, TriggerIntegrationTest, HdfsWriteToMultipleCollectionsTest, NodeLostTriggerIntegrationTest, TestCoreBackup, SpatialRPTFieldTypeTest, SolrInfoBeanTest, TestDFISimilarityFactory, TestAnalyzeInfixSuggestions, TestDistributedMap, TestSystemCollAutoCreate, TestLeaderElectionWithEmptyReplica, TestDistributedGrouping, HdfsChaosMonkeySafeLeaderTest, TestHttpShardHandlerFactory, ResponseHeaderTest, TestCryptoKeys, RestartWhileUpdatingTest, TestSimpleTrackingShardHandler, CollectionsAPIAsyncDistributedZkTest, TestEmbeddedSolrServerSchemaAPI, TestTlogReplica, TestObjectReleaseTracker, ChaosMonkeySafeLeaderTest, TimeRoutedAliasUpdateProcessorTest, TestRequestForwarding, TestComputePlanAction, TestShortCircuitedRequests, TestReqParamsAPI, JsonLoaderTest, OverriddenZkACLAndCredentialsProvidersTest, TestConfigOverlay, PolyFieldTest, CheckHdfsIndexTest, SimpleCollectionCreateDeleteTest, SoftAutoCommitTest, TestStressUserVersions, TestSchemaSimilarityResource, TestManagedSynonymGraphFilterFactory, CustomCollectionTest, SimplePostToolTest, DOMUtilTest, TestTestInjection, TestStressCloudBlindAtomicUpdates, ZkStateReaderTest, BasicFunctionalityTest, TestReloadAndDeleteDocs, DirectUpdateHandlerOptimizeTest, GraphQueryTest, MetricTriggerTest, TestGenericDistributedQueue, ReplaceNodeNoTargetTest, SolrCloudReportersTest, TestRequestStatusCollectionAPI, ExplicitHLLTest, TestCloudJSONFacetJoinDomain, JSONWriterTest, VMParamsZkACLAndCredentialsProvidersTest, TestCollapseQParserPlugin, ForceLeaderTest, TestQueryUtils, TestFieldCacheReopen, TestNodeLostTrigger, DistribCursorPagingTest, TestMultiWordSynonyms, TestSolrCloudWithDelegationTokens, ChaosMonkeySafeLeaderWithPullReplicasTest, QueryElevationComponentTest, SyncSliceTest, OpenCloseCoreStressTest, TestReplicationHandler]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestReplicationHandler -Dtests.seed=B3CB0D2BE889AEA6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=zh-CN -Dtests.timezone=Africa/Bangui -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestReplicationHandler (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [SolrCore, InternalHttpClient, MockDirectoryWrapper, MockDirectoryWrapper, MockDirectoryWrapper]
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1040)
   [junit4]    > 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:657)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1302)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:944)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:298)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:248)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:290)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1190)
   [junit4]    > 	at org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:696)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]    > 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:657)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1302)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:944)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:526)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)
   [junit4]    > 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]    > 	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
   [junit4]    > 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
   [junit4]    > 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
   [junit4]    > 	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)
   [junit4]    > 	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:730)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:955)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1047)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:643)
   [junit4]    > 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:526)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)
   [junit4]    > 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]    > 	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
   [junit4]    > 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
   [junit4]    > 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B3CB0D2BE889AEA6]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [545/794 (1!)] on J1 in 348.77s, 16 tests, 1 failure, 1 skipped <<< FAILURES!

[...truncated 50585 lines...]

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 191 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/191/

7 tests failed.
FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([A361676B624767C5:90D34FAF6FF0BD72]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:150)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:139)
	at org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter.addPosition(Lucene50PostingsWriter.java:291)
	at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:156)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:864)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343)
	at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164)
	at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:230)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:115)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4480)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4141)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2335)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5134)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1779)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1468)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:185)
	at org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit(TestDocTermOrds.java:167)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testRandom

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([A361676B624767C5:D12D4264D327D1B6]:0)
	at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
	at org.apache.lucene.util.fst.FST.<init>(FST.java:265)
	at org.apache.lucene.util.fst.Builder.<init>(Builder.java:157)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:456)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:633)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finish(BlockTreeTermsWriter.java:934)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:346)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:140)
	at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:108)
	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:163)
	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
	at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:416)
	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:473)
	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1539)
	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1512)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:155)
	at org.apache.solr.uninverting.TestDocTermOrds.testRandom(TestDocTermOrds.java:248)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testSimple

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([A361676B624767C5:9BD2439545B4B314]:0)
	at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
	at org.apache.lucene.util.fst.FST.<init>(FST.java:265)
	at org.apache.lucene.util.fst.Builder.<init>(Builder.java:157)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:456)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:633)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finish(BlockTreeTermsWriter.java:934)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:346)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:140)
	at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:108)
	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:163)
	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:454)
	at org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:392)
	at org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:328)
	at org.apache.solr.uninverting.TestDocTermOrds.testSimple(TestDocTermOrds.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testRandomWithPrefix

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([A361676B624767C5:FFE252DB0C8D8E5D]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164)
	at org.apache.lucene.store.RAMOutputStream.writeByte(RAMOutputStream.java:133)
	at org.apache.lucene.store.DataOutput.writeVInt(DataOutput.java:191)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:717)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:624)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:905)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:869)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:140)
	at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:108)
	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:163)
	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
	at org.apache.lucene.index.DocumentsWriter.flushOneDWPT(DocumentsWriter.java:262)
	at org.apache.lucene.index.IndexWriter.flushNextBuffer(IndexWriter.java:3295)
	at org.apache.lucene.index.RandomIndexWriter.maybeFlushOrCommit(RandomIndexWriter.java:203)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:188)
	at org.apache.solr.uninverting.TestDocTermOrds.testRandomWithPrefix(TestDocTermOrds.java:346)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testActuallySingleValued

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testSortedTermsEnum

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space


FAILED:  org.apache.solr.uninverting.TestDocTermOrds.testBackToTheFuture

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space




Build Log:
[...truncated 1804 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J0-20180404_131144_1022915614555193773363.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] codec: HighCompressionCompressingStoredFields, pf: BloomFilter, dvf: Direct
   [junit4] <<< JVM J0: EOF ----

[...truncated 13497 lines...]
   [junit4] Suite: org.apache.solr.uninverting.TestDocTermOrds
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrds -Dtests.method=testTriggerUnInvertLimit -Dtests.seed=A361676B624767C5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=America/North_Dakota/New_Salem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   27.9s J1 | TestDocTermOrds.testTriggerUnInvertLimit <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A361676B624767C5:90D34FAF6FF0BD72]:0)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:150)
   [junit4]    > 	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:139)
   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter.addPosition(Lucene50PostingsWriter.java:291)
   [junit4]    > 	at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:156)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:864)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343)
   [junit4]    > 	at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:230)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:115)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4480)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4141)
   [junit4]    > 	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2335)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5134)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1779)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1468)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:185)
   [junit4]    > 	at org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit(TestDocTermOrds.java:167)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrds -Dtests.method=testRandom -Dtests.seed=A361676B624767C5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=America/North_Dakota/New_Salem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   83.1s J1 | TestDocTermOrds.testRandom <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A361676B624767C5:D12D4264D327D1B6]:0)
   [junit4]    > 	at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
   [junit4]    > 	at org.apache.lucene.util.fst.FST.<init>(FST.java:265)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.<init>(Builder.java:157)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:456)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:633)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finish(BlockTreeTermsWriter.java:934)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:346)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:140)
   [junit4]    > 	at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:108)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:163)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:416)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:473)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1539)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1512)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:155)
   [junit4]    > 	at org.apache.solr.uninverting.TestDocTermOrds.testRandom(TestDocTermOrds.java:248)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrds -Dtests.method=testSimple -Dtests.seed=A361676B624767C5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=America/North_Dakota/New_Salem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   12.3s J1 | TestDocTermOrds.testSimple <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A361676B624767C5:9BD2439545B4B314]:0)
   [junit4]    > 	at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
   [junit4]    > 	at org.apache.lucene.util.fst.FST.<init>(FST.java:265)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.<init>(Builder.java:157)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:456)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:633)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finish(BlockTreeTermsWriter.java:934)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:346)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:140)
   [junit4]    > 	at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:108)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:163)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:454)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:392)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:328)
   [junit4]    > 	at org.apache.solr.uninverting.TestDocTermOrds.testSimple(TestDocTermOrds.java:110)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrds -Dtests.method=testRandomWithPrefix -Dtests.seed=A361676B624767C5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=America/North_Dakota/New_Salem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    186s J1 | TestDocTermOrds.testRandomWithPrefix <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A361676B624767C5:FFE252DB0C8D8E5D]:0)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78)
   [junit4]    > 	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164)
   [junit4]    > 	at org.apache.lucene.store.RAMOutputStream.writeByte(RAMOutputStream.java:133)
   [junit4]    > 	at org.apache.lucene.store.DataOutput.writeVInt(DataOutput.java:191)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:717)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:624)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:905)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:869)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:343)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:140)
   [junit4]    > 	at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:108)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:163)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:463)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.flushOneDWPT(DocumentsWriter.java:262)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.flushNextBuffer(IndexWriter.java:3295)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.maybeFlushOrCommit(RandomIndexWriter.java:203)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:188)
   [junit4]    > 	at org.apache.solr.uninverting.TestDocTermOrds.testRandomWithPrefix(TestDocTermOrds.java:346)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrds -Dtests.method=testActuallySingleValued -Dtests.seed=A361676B624767C5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=America/North_Dakota/New_Salem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   4.97s J1 | TestDocTermOrds.testActuallySingleValued <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrds -Dtests.method=testSortedTermsEnum -Dtests.seed=A361676B624767C5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=America/North_Dakota/New_Salem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   11.5s J1 | TestDocTermOrds.testSortedTermsEnum <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDocTermOrds -Dtests.method=testBackToTheFuture -Dtests.seed=A361676B624767C5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=America/North_Dakota/New_Salem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   62.9s J1 | TestDocTermOrds.testBackToTheFuture <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.uninverting.TestDocTermOrds_A361676B624767C5-001
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=1, maxDocsPerChunk=820, blockSize=884), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=1, blockSize=884)), sim=RandomSimilarity(queryNorm=true): {field=DFR I(F)L3(800.0)}, locale=mt, timezone=America/North_Dakota/New_Salem
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=362390336,total=469237760
   [junit4]   2> NOTE: All tests run in this JVM: [RestoreTriggerStateTest, TestLeaderElectionWithEmptyReplica, TestReplicationHandler, TestLuceneIndexBackCompat, TestStressUserVersions, TestReload, TestHdfsCloudBackupRestore, TestRecoveryHdfs, FastVectorHighlighterTest, DeleteInactiveReplicaTest, AssignTest, TestConfigOverlay, BasicAuthIntegrationTest, CdcrBootstrapTest, TestRTGBase, HdfsRecoveryZkTest, TestCryptoKeys, CoreAdminRequestStatusTest, TestDFISimilarityFactory, TestSolrCloudWithDelegationTokens, QueryEqualityTest, TestFieldCollectionResource, TestLegacyNumericRangeQueryBuilder, TestRawResponseWriter, TestFastLRUCache, TestCloudSchemaless, PolyFieldTest, SolrJmxReporterCloudTest, HighlighterTest, TestCloudRecovery, OverseerTest, SortByFunctionTest, TestDistributedMap, SearchHandlerTest, TestUniqueKeyFieldResource, PathHierarchyTokenizerFactoryTest, RuleEngineTest, MultiThreadedOCPTest, PrimitiveFieldTypeTest, HdfsDirectoryFactoryTest, DistributedQueryComponentCustomSortTest, TestGenericDistributedQueue, ZkControllerTest, TestPointFields, TestJsonFacets, SubstringBytesRefFilterTest, SimpleCollectionCreateDeleteTest, TestObjectReleaseTracker, PeerSyncReplicationTest, ForceLeaderTest, TestReqParamsAPI, BlockJoinFacetDistribTest, DeleteReplicaTest, TestScoreJoinQPNoScore, TestConfigSetsAPI, ChaosMonkeySafeLeaderTest, CheckHdfsIndexTest, TestComputePlanAction, TestFieldCacheVsDocValues, TestSQLHandlerNonCloud, TestSweetSpotSimilarityFactory, TestFieldCacheReopen, RequiredFieldsTest, TestInfoStreamLogging, ScriptEngineTest, RankQueryTest, TestPayloadCheckQParserPlugin, TestUtilizeNode, TestReversedWildcardFilterFactory, TestSolr4Spatial, DocValuesMissingTest, QueryParsingTest, AlternateDirectoryTest, SolrCoreTest, XsltUpdateRequestHandlerTest, TestCloudConsistency, VMParamsZkACLAndCredentialsProvidersTest, BigEndianAscendingWordDeserializerTest, TestAuthorizationFramework, TestSolrConfigHandler, TestFunctionQuery, LegacyCloudClusterPropTest, AnalyticsMergeStrategyTest, TestStressRecovery, HLLUtilTest, TestFreeTextSuggestions, AssignBackwardCompatibilityTest, TestXmlQParserPlugin, ShufflingReplicaListTransformerTest, TestCustomDocTransformer, ReplaceNodeNoTargetTest, ScheduledTriggerTest, TestTlogReplica, TestHttpShardHandlerFactory, SystemInfoHandlerTest, TestExactSharedStatsCache, LoggingHandlerTest, TestSystemCollAutoCreate, AutoScalingHandlerTest, TermsComponentTest, TestCopyFieldCollectionResource, PrimUtilsTest, TestSha256AuthenticationProvider, TestXIncludeConfig, MetricsHandlerTest, SaslZkACLProviderTest, TestDistributedGrouping, TestConfigSets, SampleTest, SolrInfoBeanTest, OverseerRolesTest, TestStressVersions, QueryElevationComponentTest, TestSolrIndexConfig, SegmentsInfoRequestHandlerTest, TestFuzzyAnalyzedSuggestions, DistribCursorPagingTest, TestGraphMLResponseWriter, BlockDirectoryTest, ConnectionManagerTest, TestElisionMultitermQuery, TestNumericTerms32, CollectionsAPISolrJTest, ResourceLoaderTest, DistributedFacetExistsSmallTest, TestNodeLostTrigger, TestSystemIdResolver, TestGraphTermsQParserPlugin, GraphQueryTest, RestartWhileUpdatingTest, SpatialRPTFieldTypeTest, BasicFunctionalityTest, TestZkChroot, ShardRoutingCustomTest, TestFaceting, TestRecovery, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestStressReorder, TestRangeQuery, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SuggesterFSTTest, SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTest, TestTrie, SchemaVersionSpecificBehaviorTest, DirectUpdateHandlerOptimizeTest, DistanceFunctionTest, LukeRequestHandlerTest, TestOmitPositions, DocumentBuilderTest, TestIndexingPerformance, IndexSchemaRuntimeFieldTest, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, JsonLoaderTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestLFUCache, PreAnalyzedFieldTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, DOMUtilTest, TestSolrJ, TestLRUCache, SliceStateTest, FileUtilsTest, CircularListTest, DistributedMLTComponentTest, DistributedIntervalFacetingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, TestTolerantSearch, TokenizerChainTest, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ActionThrottleTest, AddReplicaTest, AliasIntegrationTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CloudExitableDirectoryReaderTest, ConcurrentCreateRoutedAliasTest, DeleteLastCustomShardedReplicaTest, DeleteShardTest, DeleteStatusTest, DistributedQueueTest, DistributedVersionInfoTest, LIROnShardRestartTest, RecoveryAfterSoftCommitTest, SSLMigrationTest, SharedFSAutoReplicaFailoverTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudDeleteByQuery, TestCloudJSONFacetJoinDomain, TestCloudPseudoReturnFields, TestConfigSetsAPIExclusivity, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestLockTree, TestMiniSolrCloudClusterSSL, TestPullReplicaErrorHandling, TestRebalanceLeaders, TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, TestTolerantUpdateProcessorCloud, TriLevelCompositeIdRoutingTest, ZkFailoverTest, HdfsCollectionsAPIDistributedZkTest, ExecutePlanActionTest, MetricTriggerTest, NodeLostTriggerIntegrationTest, SystemLogListenerTest, TestPolicyCloud, TriggerSetPropertiesIntegrationTest, TestLargeCluster, TestNodeAddedTrigger, TestTriggerIntegration, CdcrBidirectionalTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, HDFSCollectionsAPITest, HdfsRecoverLeaseTest, HdfsThreadLeakTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsUnloadDistributedZkTest, ZkStateReaderTest, ZkStateWriterTest, BlobRepositoryCloudTest, ConfigureRecoveryStrategyTest, CoreSorterTest, DirectoryFactoryTest, QueryResultKeyTest, TestBackupRepositoryFactory, TestConfigSetProperties, TestDirectoryFactory, TestDynamicLoading, TestInitParams, TestNRTOpen, TestReloadAndDeleteDocs, TestSimpleTextCodec, TestSolrCloudSnapshots, TestConfigReload, CoreAdminOperationTest, CoreMergeIndexesAdminHandlerTest, InfoHandlerTest, PropertiesRequestHandlerTest, StatsReloadRaceTest, TestApiFramework, TestCollectionAPIs, TestConfigsApi, TestCoreAdminApis, CustomHighlightComponentTest, DistributedExpandComponentTest, DistributedFacetPivotSmallAdvancedTest, DistributedQueryComponentOptimizationTest, FacetPivotSmallTest, InfixSuggestersTest, ReplicaListTransformerTest, SpatialHeatmapFacetsTest, TestExpandComponent, JavabinLoaderTest, TestUnifiedSolrHighlighter, TestSlowCompositeReaderWrapper, UninvertDocValuesMergePolicyTest, WrapperMergePolicyFactoryTest, TestLegacyField, TestLegacyFieldReuse, TestLegacyNumericUtils, TestLegacyTerms, TestNumericRangeQuery32, TestNumericTokenStream, MetricsConfigTest, SolrCoreMetricManagerTest, SolrMetricManagerTest, SolrMetricsIntegrationTest, SolrGangliaReporterTest, SolrJmxReporterTest, SolrCloudReportersTest, RegexBytesRefFilterTest, TestIntervalFaceting, TestMissingGroups, TestReRankQParserPlugin, TestReloadDeadlock, TestSimpleQParserPlugin, TestSmileRequest, TestSolrCoreParser, TestStandardQParsers, TestTrieFacet, TestJsonFacetsWithNestedObjects, TestMinMaxOnMultiValuedField, TestOrdValues, BJQParserTest, TestCloudNestedDocsSort, TestLRUStatsCache, TestImpersonationWithHadoopAuth, HttpSolrCallGetCoreTest, ResponseHeaderTest, TestAnalyzeInfixSuggestions, BlockCacheTest, HdfsLockFactoryTest, TestDocTermOrds]
   [junit4] Completed [794/794 (1!)] on J1 in 441.58s, 10 tests, 7 errors <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J1-20180404_152752_1034969879267733083400.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid28413.hprof ...
   [junit4] Heap dump file created [467226020 bytes in 1.015 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 9287 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid28413.hprof

Total time: 213 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 190 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/190/

2 tests failed.
FAILED:  org.apache.lucene.index.TestIndexSorting.testRandom3

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([AA3475BD7E8C96D9]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexSorting

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([AA3475BD7E8C96D9]:0)




Build Log:
[...truncated 1630 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J2-20180403_085635_1795743216079533742227.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] codec: CheapBastard, pf: Lucene50, dvf: Direct
   [junit4] <<< JVM J2: EOF ----

[...truncated 53 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexSorting
   [junit4]   2> Aib 03, 2018 5:56:37 P.M. com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexSorting
   [junit4]   2>    1) Thread[id=16, name=TEST-TestIndexSorting.testRandom3-seed#[AA3475BD7E8C96D9], state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>         at sun.nio.ch.NativeThread.current(Native Method)
   [junit4]   2>         at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
   [junit4]   2>         at sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:737)
   [junit4]   2>         at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:727)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterFileChannel.read(FilterFileChannel.java:111)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterFileChannel.read(FilterFileChannel.java:111)
   [junit4]   2>         at org.apache.lucene.mockfile.FilterFileChannel.read(FilterFileChannel.java:111)
   [junit4]   2>         at org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
   [junit4]   2>         at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:269)
   [junit4]   2>         at org.apache.lucene.util.packed.DirectReader$DirectPackedReader8.get(DirectReader.java:145)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$24.nextOrd(Lucene70DocValuesProducer.java:1406)
   [junit4]   2>         at org.apache.lucene.index.AssertingLeafReader$AssertingSortedSetDocValues.nextOrd(AssertingLeafReader.java:841)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer$5$1.nextOrd(DocValuesConsumer.java:761)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedSetField(Lucene70DocValuesConsumer.java:606)
   [junit4]   2>         at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedSetField(AssertingDocValuesFormat.java:209)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4479)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4140)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2334)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5146)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1778)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1467)
   [junit4]   2>         at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2299)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2>    2) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=15, name=SUITE-TestIndexSorting-seed#[AA3475BD7E8C96D9], state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=AA3475BD7E8C96D9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ga -Dtests.timezone=Asia/Tomsk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7180s J0 | TestIndexSorting.testRandom3 <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AA3475BD7E8C96D9]:0)
   [junit4]   2> Aib 03, 2018 5:56:37 P.M. com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Aib 03, 2018 5:56:57 P.M. com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.index.TestIndexSorting: 
   [junit4]   2>    1) Thread[id=16, name=TEST-TestIndexSorting.testRandom3-seed#[AA3475BD7E8C96D9], state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$10.get(Lucene70DocValuesProducer.java:616)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$22.nextValue(Lucene70DocValuesProducer.java:1271)
   [junit4]   2>         at org.apache.lucene.index.AssertingLeafReader$AssertingSortedNumericDocValues.nextValue(AssertingLeafReader.java:752)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer$3$1.nextValue(DocValuesConsumer.java:446)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.writeValuesMultipleBlocks(Lucene70DocValuesConsumer.java:292)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.writeValues(Lucene70DocValuesConsumer.java:261)
   [junit4]   2>         at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedNumericField(Lucene70DocValuesConsumer.java:536)
   [junit4]   2>         at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedNumericField(AssertingDocValuesFormat.java:167)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
   [junit4]   2>         at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4479)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4140)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2334)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5146)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1778)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1467)
   [junit4]   2>         at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2299)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Aib 03, 2018 5:56:57 P.M. com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=16, name=TEST-TestIndexSorting.testRandom3-seed#[AA3475BD7E8C96D9], state=RUNNABLE, group=TGRP-TestIndexSorting]
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/J0/temp/lucene.index.TestIndexSorting_AA3475BD7E8C96D9-001
   [junit4]   2> Apr 03, 2018 10:56:57 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {id=DFR I(n)3(800.0)}, locale=ga, timezone=Asia/Tomsk
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=141649128,total=470810624
   [junit4]   2> NOTE: All tests run in this JVM: [TestSearch, TestSortedSetDocValues, TestIndexSorting]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexSorting -Dtests.seed=AA3475BD7E8C96D9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ga -Dtests.timezone=Asia/Tomsk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestIndexSorting (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AA3475BD7E8C96D9]:0)
   [junit4] Completed [392/468 (1!)] on J0 in 7220.18s, 14 tests, 2 errors <<< FAILURES!

[...truncated 63323 lines...]

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 189 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/189/

No tests ran.

Build Log:
[...truncated 2100 lines...]
   [junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J1-20180402_110157_705100872176170315489.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] codec: CheapBastard, pf: LuceneFixedGap, dvf: Asserting
   [junit4] <<< JVM J1: EOF ----

[...truncated 3546 lines...]
   [junit4] Suite: org.apache.lucene.index.TestBackwardsCompatibility
   [junit4] IGNOR/A 0.21s J0 | TestBackwardsCompatibility.testCreateEmptyIndex
   [junit4]    > Assumption #1: backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testAddOldIndexes -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   11.7s J0 | TestBackwardsCompatibility.testAddOldIndexes <<<
   [junit4]    > Throwable #1: java.io.FileNotFoundException: _2.si in dir=NIOFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.6.2-cfs-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@69beb36b
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:F896DF0CD3DD779E]:0)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:750)
   [junit4]    > 	at org.apache.lucene.store.Directory.copyFrom(Directory.java:159)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.copyFrom(MockDirectoryWrapper.java:1077)
   [junit4]    > 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.copyFrom(LockValidatingDirectoryWrapper.java:50)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.copySegmentAsIs(IndexWriter.java:3221)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addIndexes(IndexWriter.java:2978)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testAddOldIndexes(TestBackwardsCompatibility.java:778)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] IGNOR/A 0.04s J0 | TestBackwardsCompatibility.testCreateSortedIndex
   [junit4]    > Assumption #1: backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes
   [junit4] IGNOR/A 0.00s J0 | TestBackwardsCompatibility.testCreateMoreTermsIndex
   [junit4]    > Assumption #1: backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes
   [junit4] IGNOR/A 0.06s J0 | TestBackwardsCompatibility.testCreateCFS
   [junit4]    > Assumption #1: backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testUnsupportedOldIndexes -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.88s J0 | TestBackwardsCompatibility.testUnsupportedOldIndexes <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/2.0.0-nocfs-001/_w.frq
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:AF98162A9087B4BA]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214)
   [junit4]    > 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:129)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:129)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
   [junit4]    > 	at java.nio.file.Files.newOutputStream(Files.java:216)
   [junit4]    > 	at org.apache.lucene.util.TestUtil.unzip(TestUtil.java:152)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testUnsupportedOldIndexes(TestBackwardsCompatibility.java:647)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testMoreTerms -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   13.2s J0 | TestBackwardsCompatibility.testMoreTerms <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:4BD25FD274A88D4F]:0)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.createTempDir(LuceneTestCase.java:2754)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testMoreTerms(TestBackwardsCompatibility.java:1499)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testFullyMergeOldIndex -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.12s J0 | TestBackwardsCompatibility.testFullyMergeOldIndex <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.0.0-cfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:AFE7221290F23CD9]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1424)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1371)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testFullyMergeOldIndex(TestBackwardsCompatibility.java:738)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testIndexOldIndexNoAdds -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.02s J0 | TestBackwardsCompatibility.testIndexOldIndexNoAdds <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.0.0-cfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:1921CCF61687A05]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1424)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1371)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testIndexOldIndexNoAdds(TestBackwardsCompatibility.java:825)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testDocValuesUpdates -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   8.17s J0 | TestBackwardsCompatibility.testDocValuesUpdates <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:8E3DBDEC4F83208]:0)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.createTempDir(LuceneTestCase.java:2754)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testDocValuesUpdates(TestBackwardsCompatibility.java:1543)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] IGNOR/A 0.04s J0 | TestBackwardsCompatibility.testCreateIndexWithDocValuesUpdates
   [junit4]    > Assumption #1: backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testAddOldIndexesReader -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestBackwardsCompatibility.testAddOldIndexesReader <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.0.0-cfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:5A6638D4B1ABA612]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:655)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:627)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos.readLatestCommit(SegmentInfos.java:434)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testAddOldIndexesReader(TestBackwardsCompatibility.java:793)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testCommandLineArgs -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestBackwardsCompatibility.testCommandLineArgs <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.3.0-nocfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:657C241C66879417]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:655)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:627)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos.readLatestCommit(SegmentInfos.java:434)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testCommandLineArgs(TestBackwardsCompatibility.java:1379)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testAllIndexesUseDefaultCodec -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.02s J0 | TestBackwardsCompatibility.testAllIndexesUseDefaultCodec <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.0.0-cfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:BB1D22A9F9F6FF38]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:655)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:80)
   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:63)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.verifyUsesDefaultCodec(TestBackwardsCompatibility.java:1318)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testAllIndexesUseDefaultCodec(TestBackwardsCompatibility.java:1331)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testSortedIndex -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   6.22s J0 | TestBackwardsCompatibility.testSortedIndex <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:23FC42FF18E56A32]:0)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.createTempDir(LuceneTestCase.java:2754)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testSortedIndex(TestBackwardsCompatibility.java:1606)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testNextIntoWrongField -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.04s J0 | TestBackwardsCompatibility.testNextIntoWrongField <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.0.0-cfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:12E4F1AA4458A9DA]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:655)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:80)
   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:63)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testNextIntoWrongField(TestBackwardsCompatibility.java:1237)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testUpgradeThenMultipleCommits -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.02s J0 | TestBackwardsCompatibility.testUpgradeThenMultipleCommits <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.0.0-cfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:5869FBCD61149346]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1424)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1371)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testUpgradeThenMultipleCommits(TestBackwardsCompatibility.java:1591)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] IGNOR/A 0.00s J0 | TestBackwardsCompatibility.testCreateNoCFS
   [junit4]    > Assumption #1: backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testUpgradeOldIndex -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestBackwardsCompatibility.testUpgradeOldIndex <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.0.0-cfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:2A50259C0CFFED75]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1424)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1371)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testUpgradeOldIndex(TestBackwardsCompatibility.java:1361)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testUpgradeEmptyOldIndex -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   4.74s J0 | TestBackwardsCompatibility.testUpgradeEmptyOldIndex <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:F871505CB612690D]:0)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.createTempDir(LuceneTestCase.java:2754)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testUpgradeEmptyOldIndex(TestBackwardsCompatibility.java:1485)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.method=testIndexOldIndex -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.11s J0 | TestBackwardsCompatibility.testIndexOldIndex <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.0.0-cfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:E502BFA9A722F773]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1424)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1371)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.testIndexOldIndex(TestBackwardsCompatibility.java:836)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] IGNOR/A 0.00s J0 | TestBackwardsCompatibility.testCreateSingleSegmentNoCFS
   [junit4]    > Assumption #1: backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes
   [junit4] IGNOR/A 0.02s J0 | TestBackwardsCompatibility.testCreateSingleSegmentCFS
   [junit4]    > Assumption #1: backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=9, maxDocsPerChunk=9, blockSize=2), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=9, blockSize=2)), sim=RandomSimilarity(queryNorm=false): {content6=DFR I(F)L1, content5=DFI(Saturated), utf8=DFR I(F)Z(0.3), content=DFR I(n)B2}, locale=ca, timezone=Africa/Maseru
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=235649824,total=293076992
   [junit4]   2> NOTE: All tests run in this JVM: [TestBackwardsCompatibility]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestBackwardsCompatibility -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestBackwardsCompatibility (suite) <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J0/temp/lucene.index.TestBackwardsCompatibility_54632409201C13E6-001/6.3.0-nocfs-001
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream(UnixFileSystemProvider.java:427)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at org.apache.lucene.mockfile.ShuffleFS.newDirectoryStream(ShuffleFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:283)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newDirectoryStream(FilterFileSystemProvider.java:223)
   [junit4]    > 	at java.nio.file.Files.newDirectoryStream(Files.java:457)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:215)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1037)
   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:321)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:854)
   [junit4]    > 	at org.apache.lucene.index.TestBackwardsCompatibility.afterClass(TestBackwardsCompatibility.java:538)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [2/9 (1!)] on J0 in 116.97s, 30 tests, 16 errors, 8 skipped <<< FAILURES!

[...truncated 2 lines...]
   [junit4] Suite: org.apache.lucene.codecs.lucene50.TestLucene50SegmentInfoFormat
   [junit4] ERROR   0.00s J0 | TestLucene50SegmentInfoFormat (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.lucene.codecs.lucene50.TestLucene50SegmentInfoFormat
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4] Completed [3/9 (2!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 2 lines...]
   [junit4] Suite: org.apache.lucene.index.TestFixBrokenOffsets
   [junit4] ERROR   0.00s J0 | TestFixBrokenOffsets (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.lucene.index.TestFixBrokenOffsets
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4] Completed [4/9 (3!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 2 lines...]
   [junit4] Suite: org.apache.lucene.codecs.lucene53.TestLucene53NormsFormat
   [junit4] IGNOR/A 0.09s J1 | TestLucene53NormsFormat.testSparseShortRange
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testFewValues -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   21.7s J1 | TestLucene53NormsFormat.testFewValues <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J1/temp/lucene.codecs.lucene53.TestLucene53NormsFormat_54632409201C13E6-001/index-MMapDirectory-001/write.lock
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:76CE0B1532AD731D]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:55)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
   [junit4]    > 	at sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.readAttributes(FilterFileSystemProvider.java:177)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.readAttributes(FilterFileSystemProvider.java:177)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.readAttributes(FilterFileSystemProvider.java:177)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.readAttributes(FilterFileSystemProvider.java:177)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.readAttributes(FilterFileSystemProvider.java:177)
   [junit4]    > 	at java.nio.file.Files.readAttributes(Files.java:1737)
   [junit4]    > 	at org.apache.lucene.store.NativeFSLockFactory$NativeFSLock.ensureValid(NativeFSLockFactory.java:189)
   [junit4]    > 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:37)
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:692)
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:686)
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:660)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5073)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5058)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:562)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:502)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3340)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3554)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3519)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.commit(RandomIndexWriter.java:307)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.doTestNormsVersusDocValues(BaseNormsFormatTestCase.java:478)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testFewValues(BaseNormsFormatTestCase.java:181)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testByteRange -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   28.1s J1 | TestLucene53NormsFormat.testByteRange <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/util/Rethrow
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:9D1C1D3B408A5373]:0)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectoryImpl(LuceneTestCase.java:1655)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectoryImpl(LuceneTestCase.java:1612)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1325)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1307)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.doTestNormsVersusDocValues(BaseNormsFormatTestCase.java:452)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testByteRange(BaseNormsFormatTestCase.java:63)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.util.Rethrow
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 42 more
   [junit4] IGNOR/A 0.00s J1 | TestLucene53NormsFormat.testSparseOutliers
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4] IGNOR/A 0.00s J1 | TestLucene53NormsFormat.testSparseNCommonBig
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testFullLongRange -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.05s J1 | TestLucene53NormsFormat.testFullLongRange <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$7
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:231898A92A2D63D5]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testFullLongRange(BaseNormsFormatTestCase.java:144)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$7
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testFewLargeValues -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.02s J1 | TestLucene53NormsFormat.testFewLargeValues <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$11
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:391664E1C8249F40]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testFewLargeValues(BaseNormsFormatTestCase.java:208)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$11
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testMultiClose -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.20s J1 | TestLucene53NormsFormat.testMultiClose <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/util/packed/BlockPackedWriter
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:D05EC8323F7E7AFE]:0)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:239)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:87)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:102)
   [junit4]    > 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:404)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:251)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:497)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1775)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1467)
   [junit4]    > 	at org.apache.lucene.index.BaseIndexFileFormatTestCase.testMultiClose(BaseIndexFileFormatTestCase.java:313)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testMultiClose(BaseNormsFormatTestCase.java:52)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.util.packed.BlockPackedWriter
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 48 more
   [junit4] IGNOR/A 0.07s J1 | TestLucene53NormsFormat.testSparseFullLongRange
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testUndeadNorms -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   4.47s J1 | TestLucene53NormsFormat.testUndeadNorms <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/util/Rethrow
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:59B66064F5F2025A]:0)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectoryImpl(LuceneTestCase.java:1655)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectoryImpl(LuceneTestCase.java:1612)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1325)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1307)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:588)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> Unexpected exception: dumping fake-exception-log:...
   [junit4]   1> 
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testRandomExceptions -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.15s J1 | TestLucene53NormsFormat.testRandomExceptions <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/util/Rethrow
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:3C4C4ACDBE124A46]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseIndexFileFormatTestCase.testRandomExceptions(BaseIndexFileFormatTestCase.java:622)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testRandomExceptions(BaseNormsFormatTestCase.java:52)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.util.Rethrow
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 38 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testNCommon -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.04s J1 | TestLucene53NormsFormat.testNCommon <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$20
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:45FFCAB139C61DD9]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testNCommon(BaseNormsFormatTestCase.java:341)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$20
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testRamBytesUsed -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.04s J1 | TestLucene53NormsFormat.testRamBytesUsed <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/TieredMergePolicy$SegmentByteSizeDescending
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:A6C03649EA630CB0]:0)
   [junit4]    > 	at org.apache.lucene.index.TieredMergePolicy.findMerges(TieredMergePolicy.java:301)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updatePendingMerges(IndexWriter.java:2370)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2333)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3675)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2138)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2118)
   [junit4]    > 	at org.apache.lucene.index.BaseIndexFileFormatTestCase.testRamBytesUsed(BaseIndexFileFormatTestCase.java:275)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testRamBytesUsed(BaseNormsFormatTestCase.java:52)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] IGNOR/A 0.02s J1 | TestLucene53NormsFormat.testFewSparseValues
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testAllZeros -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.04s J1 | TestLucene53NormsFormat.testAllZeros <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$13
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:51CD12104F097BF1]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testAllZeros(BaseNormsFormatTestCase.java:234)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$13
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4] IGNOR/A 0.00s J1 | TestLucene53NormsFormat.testSparseOutliers2
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4] IGNOR/A 0.00s J1 | TestLucene53NormsFormat.testMergeStability
   [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
   [junit4] IGNOR/A 0.02s J1 | TestLucene53NormsFormat.testSparseNCommon
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testNCommonBig -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.13s J1 | TestLucene53NormsFormat.testNCommonBig <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$22
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:D96317BD4B345ADA]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testNCommonBig(BaseNormsFormatTestCase.java:390)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$22
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testLongRange -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.04s J1 | TestLucene53NormsFormat.testLongRange <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$5
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:16F14C15E4AECF38]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testLongRange(BaseNormsFormatTestCase.java:117)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$5
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testMostZeros -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.04s J1 | TestLucene53NormsFormat.testMostZeros <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$15
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:1200958C21991E96]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testMostZeros(BaseNormsFormatTestCase.java:260)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$15
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4] IGNOR/A 0.02s J1 | TestLucene53NormsFormat.testFewSparseLargeValues
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4] IGNOR/A 0.00s J1 | TestLucene53NormsFormat.testSparseAllZeros
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testShortRange -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.04s J1 | TestLucene53NormsFormat.testShortRange <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:8ABB37CC27455FB5]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testShortRange(BaseNormsFormatTestCase.java:90)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$3
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4] IGNOR/A 0.10s J1 | TestLucene53NormsFormat.testSparseByteRange
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene53NormsFormat -Dtests.method=testOutliers -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ar-BH -Dtests.timezone=Australia/Victoria -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.02s J1 | TestLucene53NormsFormat.testOutliers <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/index/BaseNormsFormatTestCase$16
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:A55ABAF88DEDB537]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testOutliers(BaseNormsFormatTestCase.java:274)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.index.BaseNormsFormatTestCase$16
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4] IGNOR/A 0.02s J1 | TestLucene53NormsFormat.testSparseLongRange
   [junit4]    > Assumption #1: Requires sparse norms support
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J1/temp/lucene.codecs.lucene53.TestLucene53NormsFormat_54632409201C13E6-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=64, maxMBSortInHeap=6.430894388234702, sim=RandomSimilarity(queryNorm=true): {foobar=IB LL-L2}, locale=ar-BH, timezone=Australia/Victoria
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=207345104,total=345505792
   [junit4]   2> NOTE: All tests run in this JVM: [TestLucene53NormsFormat]
   [junit4] Completed [5/9 (4!)] on J1 in 118.11s, 29 tests, 15 errors, 12 skipped <<< FAILURES!

[...truncated 1 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterOnOldIndex
   [junit4] ERROR   0.00s J0 | TestIndexWriterOnOldIndex (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.lucene.index.TestIndexWriterOnOldIndex
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4] Completed [6/9 (5!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 3 lines...]
   [junit4] Suite: org.apache.lucene.index.TestManyPointsInOldIndex
   [junit4] ERROR   0.00s J0 | TestManyPointsInOldIndex (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.lucene.index.TestManyPointsInOldIndex
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4] Completed [7/9 (6!)] on J0 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 2 lines...]
   [junit4] Suite: org.apache.lucene.codecs.lucene62.TestLucene62SegmentInfoFormat
   [junit4] ERROR   0.00s J1 | TestLucene62SegmentInfoFormat (suite) <<<
   [junit4]    > Throwable #1: java.lang.ClassNotFoundException: org.apache.lucene.codecs.lucene62.TestLucene62SegmentInfoFormat
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	at java.lang.Class.forName0(Native Method)
   [junit4]    > 	at java.lang.Class.forName(Class.java:348)
   [junit4] Completed [8/9 (7!)] on J1 in 0.00s, 0 tests, 1 error <<< FAILURES!

[...truncated 2 lines...]
   [junit4] Suite: org.apache.lucene.codecs.lucene54.TestLucene54DocValuesFormat
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene54DocValuesFormat -Dtests.method=testTermsEnumFixedWidth -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-BO -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   30.9s J2 | TestLucene54DocValuesFormat.testTermsEnumFixedWidth <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/J2/temp/lucene.codecs.lucene54.TestLucene54DocValuesFormat_54632409201C13E6-001/tempDir-003/_1y.fnm
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:A9639FD27A0596AA]:0)
   [junit4]    > 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]    > 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:177)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
   [junit4]    > 	at org.apache.lucene.mockfile.DisableFsyncFS.newFileChannel(DisableFsyncFS.java:45)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
   [junit4]    > 	at java.nio.channels.FileChannel.open(FileChannel.java:287)
   [junit4]    > 	at java.nio.channels.FileChannel.open(FileChannel.java:335)
   [junit4]    > 	at org.apache.lucene.util.IOUtils.fsync(IOUtils.java:460)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.fsync(FSDirectory.java:327)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.sync(FSDirectory.java:285)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.sync(FilterDirectory.java:83)
   [junit4]    > 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.sync(LockValidatingDirectoryWrapper.java:68)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4844)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3419)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3554)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3519)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.commit(RandomIndexWriter.java:307)
   [junit4]    > 	at org.apache.lucene.codecs.lucene54.TestLucene54DocValuesFormat.doTestTermsEnumRandom(TestLucene54DocValuesFormat.java:321)
   [junit4]    > 	at org.apache.lucene.codecs.lucene54.TestLucene54DocValuesFormat.testTermsEnumFixedWidth(TestLucene54DocValuesFormat.java:125)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene54DocValuesFormat -Dtests.method=testSparseByteNumericsVsStoredFields -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-BO -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   10.9s J2 | TestLucene54DocValuesFormat.testSparseByteNumericsVsStoredFields <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/util/Rethrow
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:2979A58C3C94D558]:0)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectoryImpl(LuceneTestCase.java:1655)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectoryImpl(LuceneTestCase.java:1612)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1325)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1307)
   [junit4]    > 	at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestNumericsVsStoredFields(BaseDocValuesFormatTestCase.java:1207)
   [junit4]    > 	at org.apache.lucene.index.BaseDocValuesFormatTestCase.testSparseByteNumericsVsStoredFields(BaseDocValuesFormatTestCase.java:1368)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.util.Rethrow
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 42 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene54DocValuesFormat -Dtests.method=testSortedNumericAroundBlockSize -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-BO -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.07s J2 | TestLucene54DocValuesFormat.testSortedNumericAroundBlockSize <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/document/SortedNumericDocValuesField
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:E8136807EC701F04]:0)
   [junit4]    > 	at org.apache.lucene.codecs.lucene54.TestLucene54DocValuesFormat.testSortedNumericAroundBlockSize(TestLucene54DocValuesFormat.java:608)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.document.SortedNumericDocValuesField
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 37 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene54DocValuesFormat -Dtests.method=testZeroOrMin -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-BO -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   3.04s J2 | TestLucene54DocValuesFormat.testZeroOrMin <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/codecs/lucene50/Lucene50SkipReader
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:54B254BB5C18B1F9]:0)
   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader$BlockDocsEnum.advance(Lucene50PostingsReader.java:404)
   [junit4]    > 	at org.apache.lucene.index.FilterLeafReader$FilterPostingsEnum.advance(FilterLeafReader.java:257)
   [junit4]    > 	at org.apache.lucene.index.AssertingLeafReader$AssertingPostingsEnum.advance(AssertingLeafReader.java:333)
   [junit4]    > 	at org.apache.lucene.index.CheckIndex.checkFields(CheckIndex.java:1580)
   [junit4]    > 	at org.apache.lucene.index.CheckIndex.testPostings(CheckIndex.java:1771)
   [junit4]    > 	at org.apache.lucene.util.TestUtil.checkReader(TestUtil.java:337)
   [junit4]    > 	at org.apache.lucene.util.TestUtil.checkReader(TestUtil.java:319)
   [junit4]    > 	at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestNumericsVsStoredFields(BaseDocValuesFormatTestCase.java:1253)
   [junit4]    > 	at org.apache.lucene.index.BaseDocValuesFormatTestCase.testZeroOrMin(BaseDocValuesFormatTestCase.java:2365)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.codecs.lucene50.Lucene50SkipReader
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 45 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene54DocValuesFormat -Dtests.method=testThreeFieldsMixed2 -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-BO -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.02s J2 | TestLucene54DocValuesFormat.testThreeFieldsMixed2 <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/analysis/BaseTokenStreamTestCase
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:9C79304D2234D37]:0)
   [junit4]    > 	at org.apache.lucene.analysis.MockTokenizer.<init>(MockTokenizer.java:100)
   [junit4]    > 	at org.apache.lucene.analysis.MockAnalyzer.createComponents(MockAnalyzer.java:90)
   [junit4]    > 	at org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:198)
   [junit4]    > 	at org.apache.lucene.document.Field.tokenStream(Field.java:505)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:728)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:427)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:391)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:297)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:453)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1538)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1511)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:141)
   [junit4]    > 	at org.apache.lucene.index.BaseDocValuesFormatTestCase.testThreeFieldsMixed2(BaseDocValuesFormatTestCase.java:342)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.analysis.BaseTokenStreamTestCase
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 49 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene54DocValuesFormat -Dtests.method=testSortedNumericsMultipleValuesVsStoredFields -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-BO -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.09s J2 | TestLucene54DocValuesFormat.testSortedNumericsMultipleValuesVsStoredFields <<<
   [junit4]    > Throwable #1: java.lang.NoClassDefFoundError: org/apache/lucene/document/SortedNumericDocValuesField
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54632409201C13E6:3148369840CA56E7]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseDocValuesFormatTestCase.doTestSortedNumericsVsStoredFields(BaseDocValuesFormatTestCase.java:1293)
   [junit4]    > 	at org.apache.lucene.index.BaseDocValuesFormatTestCase.testSortedNumericsMultipleValuesVsStoredFields(BaseDocValuesFormatTestCase.java:2258)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.ClassNotFoundException: org.apache.lucene.document.SortedNumericDocValuesField
   [junit4]    > 	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
   [junit4]    > 	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
   [junit4]    > 	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
   [junit4]    > 	... 38 more
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene54DocValuesFormat -Dtests.method=testThreads -Dtests.seed=54632409201C13E6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-BO -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.86s J2 | TestLucene54DocValuesFormat.testThreads <<<
   [junit4]    > Throwable #1: java.lang.

[...truncated too long message...]

es/java:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_152/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/temp/junit4-J2-20180402_143915_7848384769522635139396.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/temp/junit4-J2-20180402_143915_7848578491988967451865.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: java.io.FileNotFoundException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/temp/junit4-J2-20180402_143915_7843033552296036945390.dynamic-suites (No such file or directory)
   [junit4] 	at java.io.FileInputStream.open0(Native Method)
   [junit4] 	at java.io.FileInputStream.open(FileInputStream.java:195)
   [junit4] 	at java.io.FileInputStream.<init>(FileInputStream.java:138)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.io.Files$FileByteSource.openStream(Files.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.io.Files$FileByteSource.openStream(Files.java:116)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.io.ByteSource.copyTo(ByteSource.java:267)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1528)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] 	at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:577: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:59: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build.xml:493: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:2264: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/module-build.xml:67: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1568: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1092: At least one slave process threw an exception, first: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/test/temp/junit4-J0-20180402_143915_7838031810148709336318.dynamic-suites (No such file or directory)

Total time: 221 minutes 27 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
ERROR: Build step failed with exception
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
		at hudson.remoting.Channel.call(Channel.java:908)
		at hudson.FilePath.act(FilePath.java:986)
		at hudson.FilePath.act(FilePath.java:975)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1749)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x does not exist.
	at org.apache.tools.ant.types.AbstractFileSet.getDirectoryScanner(AbstractFileSet.java:483)
	at org.apache.tools.ant.types.AbstractFileSet.getDirectoryScanner(AbstractFileSet.java:460)
	at hudson.tasks.ArtifactArchiver$ListFiles.invoke(ArtifactArchiver.java:298)
	at hudson.tasks.ArtifactArchiver$ListFiles.invoke(ArtifactArchiver.java:278)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Build step 'Archive the artifacts' marked build as failure
Recording test results
ERROR: Build step failed with exception
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
		at hudson.remoting.Channel.call(Channel.java:908)
		at hudson.FilePath.act(FilePath.java:986)
		at hudson.FilePath.act(FilePath.java:975)
		at hudson.tasks.junit.JUnitParser.parseResult(JUnitParser.java:114)
		at hudson.tasks.junit.JUnitResultArchiver.parse(JUnitResultArchiver.java:136)
		at hudson.tasks.junit.JUnitResultArchiver.parseAndAttach(JUnitResultArchiver.java:166)
		at hudson.tasks.junit.JUnitResultArchiver.perform(JUnitResultArchiver.java:153)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1749)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x does not exist.
	at org.apache.tools.ant.types.AbstractFileSet.getDirectoryScanner(AbstractFileSet.java:483)
	at org.apache.tools.ant.types.AbstractFileSet.getDirectoryScanner(AbstractFileSet.java:460)
	at hudson.tasks.junit.JUnitParser$ParseResultCallable.invoke(JUnitParser.java:141)
	at hudson.tasks.junit.JUnitParser$ParseResultCallable.invoke(JUnitParser.java:118)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Build step 'Publish JUnit test result report' marked build as failure
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 188 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/188/

30 tests failed.
FAILED:  org.apache.solr.TestRandomDVFaceting.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:51465 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:51465 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([FB08883A2DA303F8]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:120)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:115)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:102)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:268)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:262)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
	at org.apache.solr.cloud.RecoveryZkTest.setupCluster(RecoveryZkTest.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:51465 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:232)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
	... 31 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest

Error Message:
ObjectTracker found 6 object(s) that were not released!!! [InternalHttpClient, SolrZkClient, InternalHttpClient, InternalHttpClient, SolrZkClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:509)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:321)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:113)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:519)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:509)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:503)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:321)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:113)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:519)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:503)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 6 object(s) that were not released!!! [InternalHttpClient, SolrZkClient, InternalHttpClient, InternalHttpClient, SolrZkClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:509)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:321)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:113)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:519)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:509)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)
	at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:503)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:321)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:113)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:519)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)
	at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:503)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:264)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:184)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1474)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1537)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1183)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1020)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:391)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([FB08883A2DA303F8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest

Error Message:
32 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest:     1) Thread[id=13259, name=Thread-5094, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:313)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:313)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:496)    2) Thread[id=13277, name=Scheduler-623687535, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13279, name=qtp365462096-13279, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$25/542920653.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=13282, name=qtp365462096-13282, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=13283, name=qtp365462096-13283, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=13304, name=jetty-launcher-2289-thread-1-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    7) Thread[id=13273, name=qtp392114390-13273, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$25/542920653.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=13295, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=13286, name=Scheduler-613452531, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=13272, name=qtp392114390-13272, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$25/542920653.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=13281, name=qtp365462096-13281-acceptor-0@13c51c0a-ServerConnector@5fd8a2b9{SSL,[ssl, http/1.1]}{127.0.0.1:54726}, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=13307, name=zkConnectionManagerCallback-2306-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=13285, name=qtp365462096-13285, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=13298, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=13274, name=qtp392114390-13274, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=13271, name=qtp392114390-13271-acceptor-0@4b3e6e9d-ServerConnector@21cda441{SSL,[ssl, http/1.1]}{127.0.0.1:48755}, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=13275, name=qtp392114390-13275, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=13305, name=jetty-launcher-2289-thread-2-SendThread(127.0.0.1:51465), state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)   19) Thread[id=13260, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=13270, name=qtp392114390-13270, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=13303, name=jetty-launcher-2289-thread-1-SendThread(127.0.0.1:51465), state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)   22) Thread[id=13284, name=qtp365462096-13284, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=13276, name=qtp392114390-13276, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=13308, name=zkConnectionManagerCallback-2308-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=13278, name=qtp365462096-13278, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$25/542920653.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   26) Thread[id=13265, name=ProcessThread(sid:0 cport:51465):, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:123)   27) Thread[id=13269, name=qtp392114390-13269, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=13294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   29) Thread[id=13261, name=SessionTracker, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:147)   30) Thread[id=13280, name=qtp365462096-13280, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)         at java.lang.Thread.run(Thread.java:748)   31) Thread[id=13293, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=13306, name=jetty-launcher-2289-thread-2-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 32 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest: 
   1) Thread[id=13259, name=Thread-5094, state=WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:313)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:313)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:496)
   2) Thread[id=13277, name=Scheduler-623687535, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=13279, name=qtp365462096-13279, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$25/542920653.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=13282, name=qtp365462096-13282, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=13283, name=qtp365462096-13283, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=13304, name=jetty-launcher-2289-thread-1-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   7) Thread[id=13273, name=qtp392114390-13273, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$25/542920653.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=13295, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=13286, name=Scheduler-613452531, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=13272, name=qtp392114390-13272, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$25/542920653.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=13281, name=qtp365462096-13281-acceptor-0@13c51c0a-ServerConnector@5fd8a2b9{SSL,[ssl, http/1.1]}{127.0.0.1:54726}, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=13307, name=zkConnectionManagerCallback-2306-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=13285, name=qtp365462096-13285, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=13298, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=13274, name=qtp392114390-13274, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=13271, name=qtp392114390-13271-acceptor-0@4b3e6e9d-ServerConnector@21cda441{SSL,[ssl, http/1.1]}{127.0.0.1:48755}, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=13275, name=qtp392114390-13275, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=13305, name=jetty-launcher-2289-thread-2-SendThread(127.0.0.1:51465), state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
  19) Thread[id=13260, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=13270, name=qtp392114390-13270, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  21) Thread[id=13303, name=jetty-launcher-2289-thread-1-SendThread(127.0.0.1:51465), state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
  22) Thread[id=13284, name=qtp365462096-13284, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=13276, name=qtp392114390-13276, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=13308, name=zkConnectionManagerCallback-2308-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=13278, name=qtp365462096-13278, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$25/542920653.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  26) Thread[id=13265, name=ProcessThread(sid:0 cport:51465):, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:123)
  27) Thread[id=13269, name=qtp392114390-13269, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  28) Thread[id=13294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  29) Thread[id=13261, name=SessionTracker, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:147)
  30) Thread[id=13280, name=qtp365462096-13280, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
        at java.lang.Thread.run(Thread.java:748)
  31) Thread[id=13293, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
  32) Thread[id=13306, name=jetty-launcher-2289-thread-2-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
	at __randomizedtesting.SeedInfo.seed([FB08883A2DA303F8]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=13277, name=Scheduler-623687535, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=13307, name=zkConnectionManagerCallback-2306-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13303, name=jetty-launcher-2289-thread-1-SendThread(127.0.0.1:51465), state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    4) Thread[id=13286, name=Scheduler-613452531, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=13308, name=zkConnectionManagerCallback-2308-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=13305, name=jetty-launcher-2289-thread-2-SendThread(127.0.0.1:51465), state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=13277, name=Scheduler-623687535, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=13307, name=zkConnectionManagerCallback-2306-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=13303, name=jetty-launcher-2289-thread-1-SendThread(127.0.0.1:51465), state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   4) Thread[id=13286, name=Scheduler-613452531, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=13308, name=zkConnectionManagerCallback-2308-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=13305, name=jetty-launcher-2289-thread-2-SendThread(127.0.0.1:51465), state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
	at __randomizedtesting.SeedInfo.seed([FB08883A2DA303F8]:0)


FAILED:  org.apache.solr.cloud.TestDownShardTolerantSearch.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.overseer.ZkStateWriterTest

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.core.TestShardHandlerFactory.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.core.TestSolrDeletionPolicy2.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=152260

[...truncated too long message...]

lr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/bcprov-jdk15on-1.54.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.4.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/contrib/analysis-extras/lib/icu4j-61.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_152/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20180401_162336_6553067551077345909455.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20180401_162336_6556497782831677808325.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] 	at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:577: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/common-build.xml:557: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1568: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1092: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 215 minutes 43 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #184 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

Re: [JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 187 - Still Failing

Posted by Steve Rowe <sa...@gmail.com>.
Thanks Uwe.

--
Steve
www.lucidworks.com

> On Apr 2, 2018, at 12:06 PM, Uwe Schindler <uw...@thetaphi.de> wrote:
> 
> I wrote about that yesterday.
> 
> The problem was many unused workspaces from disabled/deleted jobs. It's cleaned up now. See my mail.
> 
> Uwe
> 
> Am April 2, 2018 3:40:01 PM UTC schrieb Steve Rowe <sa...@gmail.com>:
> This is the second time the ASF Jenkins “lucene” VM has run out of disk space in the last week or so.
> 
> I did the same thing this time as I did last: I removed the workspaces for Nightly jobs, via the Jenkins web interface.
> 
> Next time I’ll try to collect more information about the situation before cleaning up. (I can log in as the jenkins account on the VM; I got OPIE working with Infra’s help.)
> 
> --
> Steve
> www.lucidworks.com
> 
>  On Mar 31, 2018, at 9:33 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
>  
>  Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/187/
>  
>  No tests ran.
>  
>  Build Log:
>  [...truncated 30 lines...]
>  ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
>  org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>  	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>  	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>  	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>  	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:134)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>  	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>  	at hudson.remoting.Request$2.run(Request.java:358)
>  	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>  	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>  	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>  	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>  	at java.lang.Thread.run(Thread.java:748)
>  org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>  	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>  	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>  	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>  Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
>  		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
>  		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
>  		at hudson.remoting.Channel.call(Channel.java:908)
>  		at hudson.FilePath.act(FilePath.java:986)
>  		at hudson.FilePath.act(FilePath.java:975)
>  		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>  		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>  		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>  		at hudson.scm.SCM.checkout(SCM.java:495)
>  		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>  		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>  		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>  		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>  		at hudson.model.Run.execute(Run.java:1724)
>  		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>  		at hudson.model.ResourceController.execute(ResourceController.java:97)
>  		at hudson.model.Executor.run(Executor.java:429)
>  Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>  	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:134)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>  	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>  	at hudson.remoting.Request$2.run(Request.java:358)
>  	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>  	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>  	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>  	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>  	at java.lang.Thread.run(Thread.java:748)
>  Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
>  	at hudson.FilePath.act(FilePath.java:993)
>  	at hudson.FilePath.act(FilePath.java:975)
>  	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>  	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>  	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>  	at hudson.scm.SCM.checkout(SCM.java:495)
>  	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>  	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>  	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>  	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>  	at hudson.model.Run.execute(Run.java:1724)
>  	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>  	at hudson.model.ResourceController.execute(ResourceController.java:97)
>  	at hudson.model.Executor.run(Executor.java:429)
>  Retrying after 10 seconds
>  git rev-parse --is-inside-work-tree # timeout=10
>  Fetching changes from the remote Git repository
>  git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
>  Fetching upstream changes from git://git.apache.org/lucene-solr.git
>  git --version # timeout=10
>  git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
>  git rev-parse refs/remotes/origin/branch_7x^{commit} # timeout=10
>  git rev-parse refs/remotes/origin/origin/branch_7x^{commit} # timeout=10
>  Checking out Revision d7486a57b155345a0aa0050adc4bcd40e3e4252c (refs/remotes/origin/branch_7x)
>  git config core.sparsecheckout # timeout=10
>  git checkout -f d7486a57b155345a0aa0050adc4bcd40e3e4252c
>  Commit message: "LUCENE-8232: Write and Checkpoint DV updates seperately if we drop a reader"
>  Cleaning workspace
>  git rev-parse --verify HEAD # timeout=10
>  Resetting working tree
>  git reset --hard # timeout=10
>  git clean -fdx # timeout=10
>  Cleaning up /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data
>  Updating http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
>  Workspace appear to be locked, so getting a fresh workspace
>  Cleaning local Directory test-data
>  Checking out http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
>  A         README.txt
>  ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
>  org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>  	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>  	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>  	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>  	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>  	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>  	at hudson.remoting.Request$2.run(Request.java:358)
>  	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>  	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>  	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>  	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>  	at java.lang.Thread.run(Thread.java:748)
>  org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>  	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>  	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>  	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>  Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
>  		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
>  		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
>  		at hudson.remoting.Channel.call(Channel.java:908)
>  		at hudson.FilePath.act(FilePath.java:986)
>  		at hudson.FilePath.act(FilePath.java:975)
>  		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>  		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>  		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>  		at hudson.scm.SCM.checkout(SCM.java:495)
>  		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>  		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>  		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>  		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>  		at hudson.model.Run.execute(Run.java:1724)
>  		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>  		at hudson.model.ResourceController.execute(ResourceController.java:97)
>  		at hudson.model.Executor.run(Executor.java:429)
>  Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>  	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>  	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>  	at hudson.remoting.Request$2.run(Request.java:358)
>  	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>  	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>  	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>  	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>  	at java.lang.Thread.run(Thread.java:748)
>  Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
>  	at hudson.FilePath.act(FilePath.java:993)
>  	at hudson.FilePath.act(FilePath.java:975)
>  	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>  	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>  	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>  	at hudson.scm.SCM.checkout(SCM.java:495)
>  	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>  	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>  	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>  	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>  	at hudson.model.Run.execute(Run.java:1724)
>  	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>  	at hudson.model.ResourceController.execute(ResourceController.java:97)
>  	at hudson.model.Executor.run(Executor.java:429)
>  Retrying after 10 seconds
>  git rev-parse --is-inside-work-tree # timeout=10
>  Fetching changes from the remote Git repository
>  git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
>  Fetching upstream changes from git://git.apache.org/lucene-solr.git
>  git --version # timeout=10
>  git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
>  git rev-parse refs/remotes/origin/branch_7x^{commit} # timeout=10
>  git rev-parse refs/remotes/origin/origin/branch_7x^{commit} # timeout=10
>  Checking out Revision d7486a57b155345a0aa0050adc4bcd40e3e4252c (refs/remotes/origin/branch_7x)
>  git config core.sparsecheckout # timeout=10
>  git checkout -f d7486a57b155345a0aa0050adc4bcd40e3e4252c
>  Commit message: "LUCENE-8232: Write and Checkpoint DV updates seperately if we drop a reader"
>  Cleaning workspace
>  git rev-parse --verify HEAD # timeout=10
>  Resetting working tree
>  git reset --hard # timeout=10
>  git clean -fdx # timeout=10
>  Cleaning up /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data
>  Updating http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
>  Workspace appear to be locked, so getting a fresh workspace
>  Cleaning local Directory test-data
>  Checking out http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
>  A         README.txt
>  ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
>  org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>  	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>  	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>  	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>  	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>  	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>  	at hudson.remoting.Request$2.run(Request.java:358)
>  	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>  	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>  	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>  	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>  	at java.lang.Thread.run(Thread.java:748)
>  org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>  	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>  	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>  	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>  	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>  	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>  	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>  	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>  Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
>  		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
>  		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
>  		at hudson.remoting.Channel.call(Channel.java:908)
>  		at hudson.FilePath.act(FilePath.java:986)
>  		at hudson.FilePath.act(FilePath.java:975)
>  		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>  		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>  		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>  		at hudson.scm.SCM.checkout(SCM.java:495)
>  		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>  		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>  		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>  		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>  		at hudson.model.Run.execute(Run.java:1724)
>  		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>  		at hudson.model.ResourceController.execute(ResourceController.java:97)
>  		at hudson.model.Executor.run(Executor.java:429)
>  Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
>  	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>  	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
>  	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>  	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>  	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>  	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>  	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>  	at hudson.remoting.Request$2.run(Request.java:358)
>  	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>  	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>  	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>  	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>  	at java.lang.Thread.run(Thread.java:748)
>  Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
>  	at hudson.FilePath.act(FilePath.java:993)
>  	at hudson.FilePath.act(FilePath.java:975)
>  	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>  	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>  	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>  	at hudson.scm.SCM.checkout(SCM.java:495)
>  	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>  	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>  	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>  	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>  	at hudson.model.Run.execute(Run.java:1724)
>  	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>  	at hudson.model.ResourceController.execute(ResourceController.java:97)
>  	at hudson.model.Executor.run(Executor.java:429)
>  Archiving artifacts
>  Recording test results
>  ERROR: Step ‘Publish JUnit test result report’ failed: No test report files were found. Configuration error?
>  Email was triggered for: Failure - Any
>  Sending email for trigger: Failure - Any
>  
> 
>  To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>  For additional commands, e-mail: dev-help@lucene.apache.org
> 
> 
> 
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
> 
> 
> --
> Uwe Schindler
> Achterdiek 19, 28357 Bremen
> https://www.thetaphi.de


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 187 - Still Failing

Posted by Uwe Schindler <uw...@thetaphi.de>.
I wrote about that yesterday.

The problem was many unused workspaces from disabled/deleted jobs. It's cleaned up now. See my mail.

Uwe

Am April 2, 2018 3:40:01 PM UTC schrieb Steve Rowe <sa...@gmail.com>:
>This is the second time the ASF Jenkins “lucene” VM has run out of disk
>space in the last week or so.
>
>I did the same thing this time as I did last: I removed the workspaces
>for Nightly jobs, via the Jenkins web interface.
>
>Next time I’ll try to collect more information about the situation
>before cleaning up. (I can log in as the jenkins account on the VM; I
>got OPIE working with Infra’s help.)
>
>--
>Steve
>www.lucidworks.com
>
>> On Mar 31, 2018, at 9:33 AM, Apache Jenkins Server
><je...@builds.apache.org> wrote:
>> 
>> Build:
>https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/187/
>> 
>> No tests ran.
>> 
>> Build Log:
>> [...truncated 30 lines...]
>> ERROR: Failed to check out
>http://svn.apache.org/repos/asf/lucene/test-data
>> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on
>device
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>> 	at
>hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:134)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>> 	at hudson.remoting.Request$2.run(Request.java:358)
>> 	at
>hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> 	at
>java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> 	at
>java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> 	at java.lang.Thread.run(Thread.java:748)
>> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on
>device
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>> Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to
>lucene
>> 		at
>hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
>> 		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
>> 		at hudson.remoting.Channel.call(Channel.java:908)
>> 		at hudson.FilePath.act(FilePath.java:986)
>> 		at hudson.FilePath.act(FilePath.java:975)
>> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>> 		at
>org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>> 		at hudson.scm.SCM.checkout(SCM.java:495)
>> 		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>> 		at
>hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>> 		at
>jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>> 		at
>hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>> 		at hudson.model.Run.execute(Run.java:1724)
>> 		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>> 		at
>hudson.model.ResourceController.execute(ResourceController.java:97)
>> 		at hudson.model.Executor.run(Executor.java:429)
>> Caused: java.io.IOException: Failed to check out
>http://svn.apache.org/repos/asf/lucene/test-data
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>> 	at
>hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:134)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>> 	at hudson.remoting.Request$2.run(Request.java:358)
>> 	at
>hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> 	at
>java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> 	at
>java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> 	at java.lang.Thread.run(Thread.java:748)
>> Caused: java.io.IOException: remote file operation failed:
>/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at
>hudson.remoting.Channel@19404e70:lucene
>> 	at hudson.FilePath.act(FilePath.java:993)
>> 	at hudson.FilePath.act(FilePath.java:975)
>> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>> 	at
>org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>> 	at hudson.scm.SCM.checkout(SCM.java:495)
>> 	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>> 	at
>hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>> 	at
>jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>> 	at
>hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>> 	at hudson.model.Run.execute(Run.java:1724)
>> 	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>> 	at
>hudson.model.ResourceController.execute(ResourceController.java:97)
>> 	at hudson.model.Executor.run(Executor.java:429)
>> Retrying after 10 seconds
>>> git rev-parse --is-inside-work-tree # timeout=10
>> Fetching changes from the remote Git repository
>>> git config remote.origin.url git://git.apache.org/lucene-solr.git #
>timeout=10
>> Fetching upstream changes from git://git.apache.org/lucene-solr.git
>>> git --version # timeout=10
>>> git fetch --tags --progress git://git.apache.org/lucene-solr.git
>+refs/heads/*:refs/remotes/origin/*
>>> git rev-parse refs/remotes/origin/branch_7x^{commit} # timeout=10
>>> git rev-parse refs/remotes/origin/origin/branch_7x^{commit} #
>timeout=10
>> Checking out Revision d7486a57b155345a0aa0050adc4bcd40e3e4252c
>(refs/remotes/origin/branch_7x)
>>> git config core.sparsecheckout # timeout=10
>>> git checkout -f d7486a57b155345a0aa0050adc4bcd40e3e4252c
>> Commit message: "LUCENE-8232: Write and Checkpoint DV updates
>seperately if we drop a reader"
>> Cleaning workspace
>>> git rev-parse --verify HEAD # timeout=10
>> Resetting working tree
>>> git reset --hard # timeout=10
>>> git clean -fdx # timeout=10
>> Cleaning up
>/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data
>> Updating http://svn.apache.org/repos/asf/lucene/test-data at revision
>'2018-03-31T13:22:46.782 +0000'
>> Workspace appear to be locked, so getting a fresh workspace
>> Cleaning local Directory test-data
>> Checking out http://svn.apache.org/repos/asf/lucene/test-data at
>revision '2018-03-31T13:22:46.782 +0000'
>> A         README.txt
>> ERROR: Failed to check out
>http://svn.apache.org/repos/asf/lucene/test-data
>> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on
>device
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>> 	at
>hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>> 	at hudson.remoting.Request$2.run(Request.java:358)
>> 	at
>hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> 	at
>java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> 	at
>java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> 	at java.lang.Thread.run(Thread.java:748)
>> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on
>device
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>> Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to
>lucene
>> 		at
>hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
>> 		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
>> 		at hudson.remoting.Channel.call(Channel.java:908)
>> 		at hudson.FilePath.act(FilePath.java:986)
>> 		at hudson.FilePath.act(FilePath.java:975)
>> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>> 		at
>org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>> 		at hudson.scm.SCM.checkout(SCM.java:495)
>> 		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>> 		at
>hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>> 		at
>jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>> 		at
>hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>> 		at hudson.model.Run.execute(Run.java:1724)
>> 		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>> 		at
>hudson.model.ResourceController.execute(ResourceController.java:97)
>> 		at hudson.model.Executor.run(Executor.java:429)
>> Caused: java.io.IOException: Failed to check out
>http://svn.apache.org/repos/asf/lucene/test-data
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>> 	at
>hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>> 	at hudson.remoting.Request$2.run(Request.java:358)
>> 	at
>hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> 	at
>java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> 	at
>java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> 	at java.lang.Thread.run(Thread.java:748)
>> Caused: java.io.IOException: remote file operation failed:
>/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at
>hudson.remoting.Channel@19404e70:lucene
>> 	at hudson.FilePath.act(FilePath.java:993)
>> 	at hudson.FilePath.act(FilePath.java:975)
>> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>> 	at
>org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>> 	at hudson.scm.SCM.checkout(SCM.java:495)
>> 	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>> 	at
>hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>> 	at
>jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>> 	at
>hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>> 	at hudson.model.Run.execute(Run.java:1724)
>> 	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>> 	at
>hudson.model.ResourceController.execute(ResourceController.java:97)
>> 	at hudson.model.Executor.run(Executor.java:429)
>> Retrying after 10 seconds
>>> git rev-parse --is-inside-work-tree # timeout=10
>> Fetching changes from the remote Git repository
>>> git config remote.origin.url git://git.apache.org/lucene-solr.git #
>timeout=10
>> Fetching upstream changes from git://git.apache.org/lucene-solr.git
>>> git --version # timeout=10
>>> git fetch --tags --progress git://git.apache.org/lucene-solr.git
>+refs/heads/*:refs/remotes/origin/*
>>> git rev-parse refs/remotes/origin/branch_7x^{commit} # timeout=10
>>> git rev-parse refs/remotes/origin/origin/branch_7x^{commit} #
>timeout=10
>> Checking out Revision d7486a57b155345a0aa0050adc4bcd40e3e4252c
>(refs/remotes/origin/branch_7x)
>>> git config core.sparsecheckout # timeout=10
>>> git checkout -f d7486a57b155345a0aa0050adc4bcd40e3e4252c
>> Commit message: "LUCENE-8232: Write and Checkpoint DV updates
>seperately if we drop a reader"
>> Cleaning workspace
>>> git rev-parse --verify HEAD # timeout=10
>> Resetting working tree
>>> git reset --hard # timeout=10
>>> git clean -fdx # timeout=10
>> Cleaning up
>/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data
>> Updating http://svn.apache.org/repos/asf/lucene/test-data at revision
>'2018-03-31T13:22:46.782 +0000'
>> Workspace appear to be locked, so getting a fresh workspace
>> Cleaning local Directory test-data
>> Checking out http://svn.apache.org/repos/asf/lucene/test-data at
>revision '2018-03-31T13:22:46.782 +0000'
>> A         README.txt
>> ERROR: Failed to check out
>http://svn.apache.org/repos/asf/lucene/test-data
>> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on
>device
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>> 	at
>hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>> 	at hudson.remoting.Request$2.run(Request.java:358)
>> 	at
>hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> 	at
>java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> 	at
>java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> 	at java.lang.Thread.run(Thread.java:748)
>> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on
>device
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
>> 	at
>org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
>> 	at
>org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
>> 	at
>org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
>> 	at
>org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
>> Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to
>lucene
>> 		at
>hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
>> 		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
>> 		at hudson.remoting.Channel.call(Channel.java:908)
>> 		at hudson.FilePath.act(FilePath.java:986)
>> 		at hudson.FilePath.act(FilePath.java:975)
>> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>> 		at
>org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>> 		at hudson.scm.SCM.checkout(SCM.java:495)
>> 		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>> 		at
>hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>> 		at
>jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>> 		at
>hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>> 		at hudson.model.Run.execute(Run.java:1724)
>> 		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>> 		at
>hudson.model.ResourceController.execute(ResourceController.java:97)
>> 		at hudson.model.Executor.run(Executor.java:429)
>> Caused: java.io.IOException: Failed to check out
>http://svn.apache.org/repos/asf/lucene/test-data
>> 	at
>hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
>> 	at
>hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
>> 	at
>hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
>> 	at
>hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
>> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
>> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
>> 	at hudson.remoting.Request$2.run(Request.java:358)
>> 	at
>hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
>> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> 	at
>java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> 	at
>java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> 	at java.lang.Thread.run(Thread.java:748)
>> Caused: java.io.IOException: remote file operation failed:
>/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at
>hudson.remoting.Channel@19404e70:lucene
>> 	at hudson.FilePath.act(FilePath.java:993)
>> 	at hudson.FilePath.act(FilePath.java:975)
>> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
>> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
>> 	at
>org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
>> 	at hudson.scm.SCM.checkout(SCM.java:495)
>> 	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
>> 	at
>hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
>> 	at
>jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
>> 	at
>hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
>> 	at hudson.model.Run.execute(Run.java:1724)
>> 	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
>> 	at
>hudson.model.ResourceController.execute(ResourceController.java:97)
>> 	at hudson.model.Executor.run(Executor.java:429)
>> Archiving artifacts
>> Recording test results
>> ERROR: Step ‘Publish JUnit test result report’ failed: No test report
>files were found. Configuration error?
>> Email was triggered for: Failure - Any
>> Sending email for trigger: Failure - Any
>> 
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>
>
>---------------------------------------------------------------------
>To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>For additional commands, e-mail: dev-help@lucene.apache.org

--
Uwe Schindler
Achterdiek 19, 28357 Bremen
https://www.thetaphi.de

Re: [JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 187 - Still Failing

Posted by Steve Rowe <sa...@gmail.com>.
This is the second time the ASF Jenkins “lucene” VM has run out of disk space in the last week or so.

I did the same thing this time as I did last: I removed the workspaces for Nightly jobs, via the Jenkins web interface.

Next time I’ll try to collect more information about the situation before cleaning up. (I can log in as the jenkins account on the VM; I got OPIE working with Infra’s help.)

--
Steve
www.lucidworks.com

> On Mar 31, 2018, at 9:33 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
> 
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/187/
> 
> No tests ran.
> 
> Build Log:
> [...truncated 30 lines...]
> ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
> 	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
> 	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
> 	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
> 	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:134)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
> 	at hudson.remoting.Request$2.run(Request.java:358)
> 	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:748)
> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
> 	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
> 	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
> 	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
> Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
> 		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
> 		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
> 		at hudson.remoting.Channel.call(Channel.java:908)
> 		at hudson.FilePath.act(FilePath.java:986)
> 		at hudson.FilePath.act(FilePath.java:975)
> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
> 		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
> 		at hudson.scm.SCM.checkout(SCM.java:495)
> 		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
> 		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
> 		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
> 		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
> 		at hudson.model.Run.execute(Run.java:1724)
> 		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
> 		at hudson.model.ResourceController.execute(ResourceController.java:97)
> 		at hudson.model.Executor.run(Executor.java:429)
> Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
> 	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:134)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
> 	at hudson.remoting.Request$2.run(Request.java:358)
> 	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:748)
> Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
> 	at hudson.FilePath.act(FilePath.java:993)
> 	at hudson.FilePath.act(FilePath.java:975)
> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
> 	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
> 	at hudson.scm.SCM.checkout(SCM.java:495)
> 	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
> 	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
> 	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
> 	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
> 	at hudson.model.Run.execute(Run.java:1724)
> 	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
> 	at hudson.model.ResourceController.execute(ResourceController.java:97)
> 	at hudson.model.Executor.run(Executor.java:429)
> Retrying after 10 seconds
>> git rev-parse --is-inside-work-tree # timeout=10
> Fetching changes from the remote Git repository
>> git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
> Fetching upstream changes from git://git.apache.org/lucene-solr.git
>> git --version # timeout=10
>> git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
>> git rev-parse refs/remotes/origin/branch_7x^{commit} # timeout=10
>> git rev-parse refs/remotes/origin/origin/branch_7x^{commit} # timeout=10
> Checking out Revision d7486a57b155345a0aa0050adc4bcd40e3e4252c (refs/remotes/origin/branch_7x)
>> git config core.sparsecheckout # timeout=10
>> git checkout -f d7486a57b155345a0aa0050adc4bcd40e3e4252c
> Commit message: "LUCENE-8232: Write and Checkpoint DV updates seperately if we drop a reader"
> Cleaning workspace
>> git rev-parse --verify HEAD # timeout=10
> Resetting working tree
>> git reset --hard # timeout=10
>> git clean -fdx # timeout=10
> Cleaning up /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data
> Updating http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
> Workspace appear to be locked, so getting a fresh workspace
> Cleaning local Directory test-data
> Checking out http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
> A         README.txt
> ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
> 	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
> 	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
> 	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
> 	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
> 	at hudson.remoting.Request$2.run(Request.java:358)
> 	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:748)
> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
> 	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
> 	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
> 	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
> Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
> 		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
> 		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
> 		at hudson.remoting.Channel.call(Channel.java:908)
> 		at hudson.FilePath.act(FilePath.java:986)
> 		at hudson.FilePath.act(FilePath.java:975)
> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
> 		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
> 		at hudson.scm.SCM.checkout(SCM.java:495)
> 		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
> 		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
> 		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
> 		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
> 		at hudson.model.Run.execute(Run.java:1724)
> 		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
> 		at hudson.model.ResourceController.execute(ResourceController.java:97)
> 		at hudson.model.Executor.run(Executor.java:429)
> Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
> 	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
> 	at hudson.remoting.Request$2.run(Request.java:358)
> 	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:748)
> Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
> 	at hudson.FilePath.act(FilePath.java:993)
> 	at hudson.FilePath.act(FilePath.java:975)
> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
> 	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
> 	at hudson.scm.SCM.checkout(SCM.java:495)
> 	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
> 	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
> 	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
> 	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
> 	at hudson.model.Run.execute(Run.java:1724)
> 	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
> 	at hudson.model.ResourceController.execute(ResourceController.java:97)
> 	at hudson.model.Executor.run(Executor.java:429)
> Retrying after 10 seconds
>> git rev-parse --is-inside-work-tree # timeout=10
> Fetching changes from the remote Git repository
>> git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
> Fetching upstream changes from git://git.apache.org/lucene-solr.git
>> git --version # timeout=10
>> git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
>> git rev-parse refs/remotes/origin/branch_7x^{commit} # timeout=10
>> git rev-parse refs/remotes/origin/origin/branch_7x^{commit} # timeout=10
> Checking out Revision d7486a57b155345a0aa0050adc4bcd40e3e4252c (refs/remotes/origin/branch_7x)
>> git config core.sparsecheckout # timeout=10
>> git checkout -f d7486a57b155345a0aa0050adc4bcd40e3e4252c
> Commit message: "LUCENE-8232: Write and Checkpoint DV updates seperately if we drop a reader"
> Cleaning workspace
>> git rev-parse --verify HEAD # timeout=10
> Resetting working tree
>> git reset --hard # timeout=10
>> git clean -fdx # timeout=10
> Cleaning up /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data
> Updating http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
> Workspace appear to be locked, so getting a fresh workspace
> Cleaning local Directory test-data
> Checking out http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
> A         README.txt
> ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
> 	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
> 	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
> 	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
> 	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
> 	at hudson.remoting.Request$2.run(Request.java:358)
> 	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:748)
> org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
> 	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
> 	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
> 	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
> 	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
> 	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
> 	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
> 	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
> Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
> 		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
> 		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
> 		at hudson.remoting.Channel.call(Channel.java:908)
> 		at hudson.FilePath.act(FilePath.java:986)
> 		at hudson.FilePath.act(FilePath.java:975)
> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
> 		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
> 		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
> 		at hudson.scm.SCM.checkout(SCM.java:495)
> 		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
> 		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
> 		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
> 		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
> 		at hudson.model.Run.execute(Run.java:1724)
> 		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
> 		at hudson.model.ResourceController.execute(ResourceController.java:97)
> 		at hudson.model.Executor.run(Executor.java:429)
> Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
> 	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
> 	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
> 	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
> 	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
> 	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
> 	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
> 	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
> 	at hudson.remoting.Request$2.run(Request.java:358)
> 	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:748)
> Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
> 	at hudson.FilePath.act(FilePath.java:993)
> 	at hudson.FilePath.act(FilePath.java:975)
> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
> 	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
> 	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
> 	at hudson.scm.SCM.checkout(SCM.java:495)
> 	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
> 	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
> 	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
> 	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
> 	at hudson.model.Run.execute(Run.java:1724)
> 	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
> 	at hudson.model.ResourceController.execute(ResourceController.java:97)
> 	at hudson.model.Executor.run(Executor.java:429)
> Archiving artifacts
> Recording test results
> ERROR: Step ‘Publish JUnit test result report’ failed: No test report files were found. Configuration error?
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 187 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/187/

No tests ran.

Build Log:
[...truncated 30 lines...]
ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:134)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
		at hudson.remoting.Channel.call(Channel.java:908)
		at hudson.FilePath.act(FilePath.java:986)
		at hudson.FilePath.act(FilePath.java:975)
		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
		at hudson.scm.SCM.checkout(SCM.java:495)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1724)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:134)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
	at hudson.FilePath.act(FilePath.java:993)
	at hudson.FilePath.act(FilePath.java:975)
	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
	at hudson.scm.SCM.checkout(SCM.java:495)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1724)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
Fetching upstream changes from git://git.apache.org/lucene-solr.git
 > git --version # timeout=10
 > git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
 > git rev-parse refs/remotes/origin/branch_7x^{commit} # timeout=10
 > git rev-parse refs/remotes/origin/origin/branch_7x^{commit} # timeout=10
Checking out Revision d7486a57b155345a0aa0050adc4bcd40e3e4252c (refs/remotes/origin/branch_7x)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f d7486a57b155345a0aa0050adc4bcd40e3e4252c
Commit message: "LUCENE-8232: Write and Checkpoint DV updates seperately if we drop a reader"
Cleaning workspace
 > git rev-parse --verify HEAD # timeout=10
Resetting working tree
 > git reset --hard # timeout=10
 > git clean -fdx # timeout=10
Cleaning up /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data
Updating http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
Workspace appear to be locked, so getting a fresh workspace
Cleaning local Directory test-data
Checking out http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
A         README.txt
ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
		at hudson.remoting.Channel.call(Channel.java:908)
		at hudson.FilePath.act(FilePath.java:986)
		at hudson.FilePath.act(FilePath.java:975)
		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
		at hudson.scm.SCM.checkout(SCM.java:495)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1724)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
	at hudson.FilePath.act(FilePath.java:993)
	at hudson.FilePath.act(FilePath.java:975)
	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
	at hudson.scm.SCM.checkout(SCM.java:495)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1724)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
Fetching upstream changes from git://git.apache.org/lucene-solr.git
 > git --version # timeout=10
 > git fetch --tags --progress git://git.apache.org/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
 > git rev-parse refs/remotes/origin/branch_7x^{commit} # timeout=10
 > git rev-parse refs/remotes/origin/origin/branch_7x^{commit} # timeout=10
Checking out Revision d7486a57b155345a0aa0050adc4bcd40e3e4252c (refs/remotes/origin/branch_7x)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f d7486a57b155345a0aa0050adc4bcd40e3e4252c
Commit message: "LUCENE-8232: Write and Checkpoint DV updates seperately if we drop a reader"
Cleaning workspace
 > git rev-parse --verify HEAD # timeout=10
Resetting working tree
 > git reset --hard # timeout=10
 > git clean -fdx # timeout=10
Cleaning up /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data
Updating http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
Workspace appear to be locked, so getting a fresh workspace
Cleaning local Directory test-data
Checking out http://svn.apache.org/repos/asf/lucene/test-data at revision '2018-03-31T13:22:46.782 +0000'
A         README.txt
ERROR: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
org.tmatesoft.svn.core.SVNException: svn: E204899: No space left on device
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:70)
	at org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:57)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:798)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:391)
	at org.tmatesoft.svn.core.internal.io.dav.http.HTTPConnection.request(HTTPConnection.java:379)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.performHttpRequest(DAVConnection.java:862)
	at org.tmatesoft.svn.core.internal.io.dav.DAVConnection.doReport(DAVConnection.java:359)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.runReport(DAVRepository.java:1341)
	at org.tmatesoft.svn.core.internal.io.dav.DAVRepository.update(DAVRepository.java:868)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.updateInternal(SvnNgAbstractUpdate.java:247)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.update(SvnNgAbstractUpdate.java:111)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgAbstractUpdate.checkout(SvnNgAbstractUpdate.java:861)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:26)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgCheckout.run(SvnNgCheckout.java:11)
	at org.tmatesoft.svn.core.internal.wc2.ng.SvnNgOperationRunner.run(SvnNgOperationRunner.java:20)
	at org.tmatesoft.svn.core.internal.wc2.SvnOperationRunner.run(SvnOperationRunner.java:21)
	at org.tmatesoft.svn.core.wc2.SvnOperationFactory.run(SvnOperationFactory.java:1235)
	at org.tmatesoft.svn.core.wc2.SvnOperation.run(SvnOperation.java:294)
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:120)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
		at hudson.remoting.Channel.call(Channel.java:908)
		at hudson.FilePath.act(FilePath.java:986)
		at hudson.FilePath.act(FilePath.java:975)
		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
		at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
		at hudson.scm.SCM.checkout(SCM.java:495)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1724)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: java.io.IOException: Failed to check out http://svn.apache.org/repos/asf/lucene/test-data
	at hudson.scm.subversion.CheckoutUpdater$1.perform(CheckoutUpdater.java:131)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:176)
	at hudson.scm.subversion.UpdateUpdater$TaskImpl.perform(UpdateUpdater.java:186)
	at hudson.scm.subversion.WorkspaceUpdater$UpdateTask.delegateTo(WorkspaceUpdater.java:168)
	at hudson.scm.SubversionSCM$CheckOutTask.perform(SubversionSCM.java:1030)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:1006)
	at hudson.scm.SubversionSCM$CheckOutTask.invoke(SubversionSCM.java:979)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2760)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.io.IOException: remote file operation failed: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x at hudson.remoting.Channel@19404e70:lucene
	at hudson.FilePath.act(FilePath.java:993)
	at hudson.FilePath.act(FilePath.java:975)
	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:928)
	at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:864)
	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
	at hudson.scm.SCM.checkout(SCM.java:495)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1724)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Archiving artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: No test report files were found. Configuration error?
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 186 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/186/

No tests ran.

Build Log:
[...truncated 6 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:862)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1129)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1160)
	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
	at hudson.scm.SCM.checkout(SCM.java:495)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1724)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1996)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1964)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1960)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor239.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:922)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:896)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:853)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
		at hudson.remoting.Channel.call(Channel.java:908)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:281)
		at com.sun.proxy.$Proxy110.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:850)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1129)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1160)
		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
		at hudson.scm.SCM.checkout(SCM.java:495)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1724)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:862)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1129)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1160)
	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
	at hudson.scm.SCM.checkout(SCM.java:495)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1724)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1996)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1964)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1960)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor239.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:922)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:896)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:853)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
		at hudson.remoting.Channel.call(Channel.java:908)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:281)
		at com.sun.proxy.$Proxy110.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:850)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1129)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1160)
		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
		at hudson.scm.SCM.checkout(SCM.java:495)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1724)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
	at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:862)
	at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1129)
	at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1160)
	at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
	at hudson.scm.SCM.checkout(SCM.java:495)
	at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
	at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
	at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
	at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
	at hudson.model.Run.execute(Run.java:1724)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout: 
stderr: error: failed to write new configuration file .git/config.lock

	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1996)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1964)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1960)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
	at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
	at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
	at sun.reflect.GeneratedMethodAccessor239.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:922)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:896)
	at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:853)
	at hudson.remoting.UserRequest.perform(UserRequest.java:207)
	at hudson.remoting.UserRequest.perform(UserRequest.java:53)
	at hudson.remoting.Request$2.run(Request.java:358)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1693)
		at hudson.remoting.UserResponse.retrieve(UserRequest.java:310)
		at hudson.remoting.Channel.call(Channel.java:908)
		at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:281)
		at com.sun.proxy.$Proxy110.setRemoteUrl(Unknown Source)
		at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
		at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:850)
		at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1129)
		at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1160)
		at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
		at hudson.scm.SCM.checkout(SCM.java:495)
		at hudson.model.AbstractProject.checkout(AbstractProject.java:1202)
		at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
		at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
		at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
		at hudson.model.Run.execute(Run.java:1724)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #184 to compare, so performing full copy of artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run? 
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/common/test/TEST-org.apache.lucene.analysis.ar.TestArabicAnalyzer.xml is 3 hr 19 min old

Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 185 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/185/

24 tests failed.
FAILED:  org.apache.lucene.index.Test4GBStoredFields.test

Error Message:
No space left on device

Stack Trace:
java.io.IOException: No space left on device
	at __randomizedtesting.SeedInfo.seed([4D655CBF0A6A114A:C5316365A4967CB2]:0)
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
	at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
	at sun.nio.ch.IOUtil.write(IOUtil.java:65)
	at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
	at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
	at java.nio.channels.Channels.writeFully(Channels.java:101)
	at java.nio.channels.Channels.access$000(Channels.java:61)
	at java.nio.channels.Channels$1.write(Channels.java:174)
	at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
	at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
	at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
	at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
	at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
	at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53)
	at org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
	at org.apache.lucene.store.MockIndexOutputWrapper.copyBytes(MockIndexOutputWrapper.java:165)
	at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:100)
	at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5051)
	at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:562)
	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:502)
	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556)
	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672)
	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3696)
	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3671)
	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2135)
	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2115)
	at org.apache.lucene.index.Test4GBStoredFields.test(Test4GBStoredFields.java:90)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: java.io.IOException: No space left on device
		at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
		at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
		at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
		at sun.nio.ch.IOUtil.write(IOUtil.java:65)
		at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
		at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
		at java.nio.channels.Channels.writeFully(Channels.java:101)
		at java.nio.channels.Channels.access$000(Channels.java:61)
		at java.nio.channels.Channels$1.write(Channels.java:174)
		at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
		at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
		at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
		at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
		at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
		at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
		at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68)
		at org.apache.lucene.store.MockIndexOutputWrapper.close(MockIndexOutputWrapper.java:104)
		at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:123)
		... 46 more
		Suppressed: java.io.IOException: No space left on device
			at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
			at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
			at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
			at sun.nio.ch.IOUtil.write(IOUtil.java:65)
			at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
			at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
			at java.nio.channels.Channels.writeFully(Channels.java:101)
			at java.nio.channels.Channels.access$000(Channels.java:61)
			at java.nio.channels.Channels$1.write(Channels.java:174)
			at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
			at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
			at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
			at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
			at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
			at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
			at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
			at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70)
			... 48 more


FAILED:  org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([4D655CBF0A6A114A]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([4D655CBF0A6A114A]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager

Error Message:
Captured an uncaught exception in thread: Thread[id=2820, name=Thread-2640, state=RUNNABLE, group=TGRP-TestSearcherManager]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2820, name=Thread-2640, state=RUNNABLE, group=TGRP-TestSearcherManager]
Caused by: java.lang.RuntimeException: java.io.IOException: No space left on device
	at __randomizedtesting.SeedInfo.seed([4D655CBF0A6A114A]:0)
	at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:590)
Caused by: java.io.IOException: No space left on device
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
	at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
	at sun.nio.ch.IOUtil.write(IOUtil.java:65)
	at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
	at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
	at java.nio.channels.Channels.writeFully(Channels.java:101)
	at java.nio.channels.Channels.access$000(Channels.java:61)
	at java.nio.channels.Channels$1.write(Channels.java:174)
	at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
	at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
	at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
	at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
	at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
	at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53)
	at org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
	at org.apache.lucene.store.MockIndexOutputWrapper.copyBytes(MockIndexOutputWrapper.java:165)
	at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:100)
	at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5051)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4541)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4138)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2331)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5144)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1775)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1463)
	at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:574)
	Suppressed: java.io.IOException: No space left on device
		at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
		at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
		at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
		at sun.nio.ch.IOUtil.write(IOUtil.java:65)
		at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
		at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
		at java.nio.channels.Channels.writeFully(Channels.java:101)
		at java.nio.channels.Channels.access$000(Channels.java:61)
		at java.nio.channels.Channels$1.write(Channels.java:174)
		at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
		at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
		at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
		at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
		at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
		at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
		at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68)
		at org.apache.lucene.store.MockIndexOutputWrapper.close(MockIndexOutputWrapper.java:104)
		at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:123)
		... 9 more
		Suppressed: java.io.IOException: No space left on device
			at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
			at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
			at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
			at sun.nio.ch.IOUtil.write(IOUtil.java:65)
			at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
			at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
			at java.nio.channels.Channels.writeFully(Channels.java:101)
			at java.nio.channels.Channels.access$000(Channels.java:61)
			at java.nio.channels.Channels$1.write(Channels.java:174)
			at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
			at org.apache.lucene.mockfile.FilterOutputStream2.write(FilterOutputStream2.java:62)
			at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
			at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
			at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
			at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
			at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
			at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70)
			... 11 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space


FAILED:  org.apache.solr.cloud.MoveReplicaHDFSTest.test

Error Message:
expected:<100> but was:<56>

Stack Trace:
java.lang.AssertionError: expected:<100> but was:<56>
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:B9ACE668F1D73D30]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.MoveReplicaTest.test(MoveReplicaTest.java:200)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes

Error Message:
Could not find collection : nodes_used_collection

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : nodes_used_collection
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:55B80CDEE4E5D8D9]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:118)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes(CollectionsAPIDistributedZkTest.java:344)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.moveReplicaTest

Error Message:
Collection not found: movereplicatest_coll

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: movereplicatest_coll
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:36DA612808B1E18A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:852)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.moveReplicaTest(HdfsCollectionsAPIDistributedZkTest.java:96)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testSpecificConfigsets

Error Message:
Error from server at http://127.0.0.1:44134/solr: create the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: create the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:1C8696E8A80FEAC4]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testSpecificConfigsets(CollectionsAPIDistributedZkTest.java:371)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection

Error Message:
Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:D6DFC98F48482F31]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testMissingRequiredParameters

Error Message:
Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:7E3E28D4C2A26769]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.deleteCollectionRemovesStaleZkCollectionsNode

Error Message:
Error from server at http://127.0.0.1:44134/solr: Could not find collection : withconfigset2

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: Could not find collection : withconfigset2
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:B351C9D371706C55]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.addReplicaTest

Error Message:
Error from server at http://127.0.0.1:44134/solr: create the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: create the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:A2588E94D1231B19]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.addReplicaTest(CollectionsAPIDistributedZkTest.java:620)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testCreationAndDeletion

Error Message:
Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:3FC7862543F0369]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testMissingNumShards

Error Message:
Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:39A870131475D0AE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testCollectionReload

Error Message:
Error from server at http://127.0.0.1:44134/solr: Could not find collection : addReplicaColl

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: Could not find collection : addReplicaColl
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:3041F58C8AA4FF57]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testCreateShouldFailOnExistingCore

Error Message:
Error from server at http://127.0.0.1:44134/solr: addreplica the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: addreplica the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:DD3E7406A4B7ED2C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCreateShouldFailOnExistingCore(CollectionsAPIDistributedZkTest.java:291)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testCreateNodeSet

Error Message:
Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:6A5B0E5236AA7440]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.deleteCollectionOnlyInZk

Error Message:
Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: delete the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:85188533A92DF0D9]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.GeneratedMethodAccessor96.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection

Error Message:
Error from server at http://127.0.0.1:44134/solr: Could not find collection : halfcollectionblocker

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44134/solr: Could not find collection : halfcollectionblocker
	at __randomizedtesting.SeedInfo.seed([31F8D9B25F2B50C8:1B78B557E8F17000]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:451)
	at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.clearCluster(CollectionsAPIDistributedZkTest.java:114)
	at sun.reflect.GeneratedMethodAccessor96.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.

[...truncated too long message...]

lr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/bcprov-jdk15on-1.54.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.4.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/contrib/analysis-extras/lib/icu4j-61.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20180328_200105_3083583459994882171710.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20180328_200105_3088563612136275280786.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] 	at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:577: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/common-build.xml:557: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1568: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1092: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 269 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #184 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 184 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/184/

1 tests failed.
FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchWithMasterUrl

Error Message:
expected:<null> but was:<.numFound:505!=501>

Stack Trace:
java.lang.AssertionError: expected:<null> but was:<.numFound:505!=501>
	at __randomizedtesting.SeedInfo.seed([D615EA42C65A9D66:CDAE1B3DA7E5E5D0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchWithMasterUrl(TestReplicationHandler.java:800)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 2078 lines...]
   [junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/core/test/temp/junit4-J0-20180327_105639_6958882232015759046116.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] codec: FastDecompressionCompressingStoredFields, pf: FST50, dvf: Lucene70
   [junit4] <<< JVM J0: EOF ----

[...truncated 11494 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/init-core-data-001
   [junit4]   2> 1922676 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T21:27:37Z, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1922684 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1922684 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1922684 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session Scavenging every 660000ms
   [junit4]   2> 1922685 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@478b5648{/solr,null,AVAILABLE}
   [junit4]   2> 1922753 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2296cf90{HTTP/1.1,[http/1.1]}{127.0.0.1:43002}
   [junit4]   2> 1922753 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.Server Started @1922872ms
   [junit4]   2> 1922753 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=43002}
   [junit4]   2> 1922753 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1922754 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1922754 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1922754 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1922754 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1922754 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-27T16:54:33.440Z
   [junit4]   2> 1922754 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-001/solr.xml
   [junit4]   2> 1922758 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1922758 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1922759 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1922903 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e
   [junit4]   2> 1922941 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e
   [junit4]   2> 1922941 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e
   [junit4]   2> 1922956 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-001/.
   [junit4]   2> 1922956 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1922973 INFO  (coreLoadExecutor-2166-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1923042 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 1923141 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1923157 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1923157 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-001/./collection1, trusted=true
   [junit4]   2> 1923157 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e
   [junit4]   2> 1923157 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1923157 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-001/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1923176 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8730183456958045]
   [junit4]   2> 1923327 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1923327 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1923327 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=84.43359375, floorSegmentMB=1.845703125, forceMergeDeletesPctAllowed=9.695715774566482, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44599295710442666
   [junit4]   2> 1923329 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3ebceba[collection1] main]
   [junit4]   2> 1923329 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-001/collection1/conf
   [junit4]   2> 1923342 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 1923342 INFO  (coreLoadExecutor-2166-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1923343 INFO  (searcherExecutor-2167-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3ebceba[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1923356 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T21:27:37Z, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
   [junit4]   2> 1923357 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1923357 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1923357 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session Scavenging every 600000ms
   [junit4]   2> 1923357 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@21686127{/solr,null,AVAILABLE}
   [junit4]   2> 1923366 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7d2365d0{HTTP/1.1,[http/1.1]}{127.0.0.1:46173}
   [junit4]   2> 1923366 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.Server Started @1923486ms
   [junit4]   2> 1923366 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=46173}
   [junit4]   2> 1923366 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1923366 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1923366 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.4.0
   [junit4]   2> 1923367 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1923367 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1923367 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-03-27T16:54:34.053Z
   [junit4]   2> 1923367 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-002/solr.xml
   [junit4]   2> 1923383 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1923383 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1923384 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1923491 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e
   [junit4]   2> 1923563 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e
   [junit4]   2> 1923563 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e
   [junit4]   2> 1923578 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-002/.
   [junit4]   2> 1923578 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1923578 INFO  (coreLoadExecutor-2176-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1923609 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.4.0
   [junit4]   2> 1923644 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1923646 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1923646 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-002/./collection1, trusted=true
   [junit4]   2> 1923658 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1397ff1e
   [junit4]   2> 1923658 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1923658 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-002/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 1923661 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8730183456958045]
   [junit4]   2> 1923906 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1923906 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1923908 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=84.43359375, floorSegmentMB=1.845703125, forceMergeDeletesPctAllowed=9.695715774566482, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44599295710442666
   [junit4]   2> 1923909 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@dfb619d[collection1] main]
   [junit4]   2> 1923909 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001/solr-instance-002/collection1/conf
   [junit4]   2> 1923912 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
   [junit4]   2> 1923912 INFO  (coreLoadExecutor-2176-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1923913 INFO  (searcherExecutor-2177-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@dfb619d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1923935 INFO  (qtp539172872-5132) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1923938 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 1
   [junit4]   2> 1923938 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1]} 0 0
   [junit4]   2> 1923951 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 0
   [junit4]   2> 1923951 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[3]} 0 0
   [junit4]   2> 1923952 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[4]} 0 0
   [junit4]   2> 1923952 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[5]} 0 0
   [junit4]   2> 1923952 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[6]} 0 0
   [junit4]   2> 1923953 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[7]} 0 0
   [junit4]   2> 1923953 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[8]} 0 0
   [junit4]   2> 1923953 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[9]} 0 0
   [junit4]   2> 1923954 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[10]} 0 0
   [junit4]   2> 1923954 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[11]} 0 0
   [junit4]   2> 1923954 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[12]} 0 0
   [junit4]   2> 1923971 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[13]} 0 0
   [junit4]   2> 1923972 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[14]} 0 0
   [junit4]   2> 1923972 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[15]} 0 0
   [junit4]   2> 1923972 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[16]} 0 0
   [junit4]   2> 1923973 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[17]} 0 0
   [junit4]   2> 1923973 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[18]} 0 0
   [junit4]   2> 1923973 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[19]} 0 0
   [junit4]   2> 1923974 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[20]} 0 0
   [junit4]   2> 1923974 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[21]} 0 0
   [junit4]   2> 1923974 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[22]} 0 0
   [junit4]   2> 1923974 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[23]} 0 0
   [junit4]   2> 1923975 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[24]} 0 0
   [junit4]   2> 1923975 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[25]} 0 0
   [junit4]   2> 1923975 INFO  (qtp539172872-5134) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[26]} 0 0
   [junit4]   2> 1923976 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[27]} 0 0
   [junit4]   2> 1923976 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[28]} 0 0
   [junit4]   2> 1923977 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[29]} 0 0
   [junit4]   2> 1923977 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[30]} 0 0
   [junit4]   2> 1923978 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[31]} 0 0
   [junit4]   2> 1923978 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[32]} 0 0
   [junit4]   2> 1923979 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[33]} 0 0
   [junit4]   2> 1923991 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[34]} 0 0
   [junit4]   2> 1923992 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[35]} 0 0
   [junit4]   2> 1923992 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[36]} 0 0
   [junit4]   2> 1923992 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[37]} 0 0
   [junit4]   2> 1923993 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[38]} 0 0
   [junit4]   2> 1923993 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[39]} 0 0
   [junit4]   2> 1923994 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[40]} 0 0
   [junit4]   2> 1923994 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[41]} 0 0
   [junit4]   2> 1923994 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[42]} 0 0
   [junit4]   2> 1923994 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[43]} 0 0
   [junit4]   2> 1923995 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[44]} 0 0
   [junit4]   2> 1923995 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[45]} 0 0
   [junit4]   2> 1923995 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[46]} 0 0
   [junit4]   2> 1923996 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[47]} 0 0
   [junit4]   2> 1923996 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[48]} 0 0
   [junit4]   2> 1923997 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[49]} 0 0
   [junit4]   2> 1923997 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[50]} 0 0
   [junit4]   2> 1923998 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[51]} 0 0
   [junit4]   2> 1923998 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[52]} 0 0
   [junit4]   2> 1923998 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[53]} 0 0
   [junit4]   2> 1923999 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[54]} 0 0
   [junit4]   2> 1923999 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[55]} 0 0
   [junit4]   2> 1923999 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[56]} 0 0
   [junit4]   2> 1924012 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[57]} 0 0
   [junit4]   2> 1924012 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[58]} 0 0
   [junit4]   2> 1924013 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[59]} 0 0
   [junit4]   2> 1924013 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[60]} 0 0
   [junit4]   2> 1924013 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[61]} 0 0
   [junit4]   2> 1924014 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[62]} 0 0
   [junit4]   2> 1924014 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[63]} 0 0
   [junit4]   2> 1924014 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[64]} 0 0
   [junit4]   2> 1924015 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[65]} 0 0
   [junit4]   2> 1924015 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[66]} 0 0
   [junit4]   2> 1924024 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[67]} 0 8
   [junit4]   2> 1924024 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[68]} 0 0
   [junit4]   2> 1924025 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[69]} 0 0
   [junit4]   2> 1924025 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[70]} 0 0
   [junit4]   2> 1924025 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[71]} 0 0
   [junit4]   2> 1924025 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[72]} 0 0
   [junit4]   2> 1924026 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[73]} 0 0
   [junit4]   2> 1924026 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[74]} 0 0
   [junit4]   2> 1924026 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[75]} 0 0
   [junit4]   2> 1924027 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[76]} 0 0
   [junit4]   2> 1924027 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[77]} 0 0
   [junit4]   2> 1924027 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[78]} 0 0
   [junit4]   2> 1924028 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[79]} 0 0
   [junit4]   2> 1924040 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[80]} 0 0
   [junit4]   2> 1924041 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[81]} 0 0
   [junit4]   2> 1924042 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[82]} 0 0
   [junit4]   2> 1924042 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[83]} 0 0
   [junit4]   2> 1924043 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[84]} 0 0
   [junit4]   2> 1924044 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[85]} 0 0
   [junit4]   2> 1924044 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[86]} 0 0
   [junit4]   2> 1924044 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[87]} 0 0
   [junit4]   2> 1924045 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[88]} 0 0
   [junit4]   2> 1924045 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[89]} 0 0
   [junit4]   2> 1924045 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[90]} 0 0
   [junit4]   2> 1924046 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[91]} 0 0
   [junit4]   2> 1924046 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[92]} 0 0
   [junit4]   2> 1924046 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[93]} 0 0
   [junit4]   2> 1924046 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[94]} 0 0
   [junit4]   2> 1924047 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[95]} 0 0
   [junit4]   2> 1924047 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[96]} 0 0
   [junit4]   2> 1924047 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[97]} 0 0
   [junit4]   2> 1924048 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[98]} 0 0
   [junit4]   2> 1924061 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[99]} 0 12
   [junit4]   2> 1924061 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[100]} 0 0
   [junit4]   2> 1924061 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[101]} 0 0
   [junit4]   2> 1924061 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[102]} 0 0
   [junit4]   2> 1924062 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[103]} 0 0
   [junit4]   2> 1924062 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[104]} 0 0
   [junit4]   2> 1924062 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[105]} 0 0
   [junit4]   2> 1924063 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[106]} 0 0
   [junit4]   2> 1924063 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[107]} 0 0
   [junit4]   2> 1924063 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[108]} 0 0
   [junit4]   2> 1924064 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[109]} 0 0
   [junit4]   2> 1924064 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[110]} 0 0
   [junit4]   2> 1924064 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[111]} 0 0
   [junit4]   2> 1924081 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[112]} 0 0
   [junit4]   2> 1924081 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[113]} 0 0
   [junit4]   2> 1924082 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[114]} 0 0
   [junit4]   2> 1924082 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[115]} 0 0
   [junit4]   2> 1924082 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[116]} 0 0
   [junit4]   2> 1924083 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[117]} 0 0
   [junit4]   2> 1924083 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[118]} 0 0
   [junit4]   2> 1924083 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[119]} 0 0
   [junit4]   2> 1924084 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[120]} 0 0
   [junit4]   2> 1924084 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[121]} 0 0
   [junit4]   2> 1924084 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[122]} 0 0
   [junit4]   2> 1924097 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[123]} 0 0
   [junit4]   2> 1924098 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[124]} 0 0
   [junit4]   2> 1924098 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[125]} 0 0
   [junit4]   2> 1924098 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[126]} 0 0
   [junit4]   2> 1924099 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[127]} 0 0
   [junit4]   2> 1924099 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[128]} 0 0
   [junit4]   2> 1924100 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[129]} 0 0
   [junit4]   2> 1924100 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[130]} 0 0
   [junit4]   2> 1924100 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[131]} 0 0
   [junit4]   2> 1924100 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[132]} 0 0
   [junit4]   2> 1924101 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[133]} 0 0
   [junit4]   2> 1924118 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[134]} 0 0
   [junit4]   2> 1924118 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[135]} 0 0
   [junit4]   2> 1924118 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[136]} 0 0
   [junit4]   2> 1924119 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[137]} 0 0
   [junit4]   2> 1924119 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[138]} 0 0
   [junit4]   2> 1924120 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[139]} 0 0
   [junit4]   2> 1924120 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[140]} 0 0
   [junit4]   2> 1924120 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[141]} 0 0
   [junit4]   2> 1924121 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[142]} 0 0
   [junit4]   2> 1924121 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[143]} 0 0
   [junit4]   2> 1924134 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[144]} 0 0
   [junit4]   2> 1924134 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[145]} 0 0
   [junit4]   2> 1924135 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[146]} 0 0
   [junit4]   2> 1924135 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[147]} 0 0
   [junit4]   2> 1924136 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[148]} 0 0
   [junit4]   2> 1924136 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[149]} 0 0
   [junit4]   2> 1924137 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[150]} 0 0
   [junit4]   2> 1924137 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[151]} 0 0
   [junit4]   2> 1924138 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[152]} 0 0
   [junit4]   2> 1924138 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[153]} 0 0
   [junit4]   2> 1924138 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[154]} 0 0
   [junit4]   2> 1924139 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[155]} 0 0
   [junit4]   2> 1924139 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[156]} 0 0
   [junit4]   2> 1924139 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[157]} 0 0
   [junit4]   2> 1924139 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[158]} 0 0
   [junit4]   2> 1924140 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[159]} 0 0
   [junit4]   2> 1924140 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[160]} 0 0
   [junit4]   2> 1924140 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[161]} 0 0
   [junit4]   2> 1924141 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[162]} 0 0
   [junit4]   2> 1924141 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[163]} 0 0
   [junit4]   2> 1924141 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[164]} 0 0
   [junit4]   2> 1924141 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[165]} 0 0
   [junit4]   2> 1924162 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[166]} 0 0
   [junit4]   2> 1924163 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[167]} 0 0
   [junit4]   2> 1924163 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[168]} 0 0
   [junit4]   2> 1924163 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[169]} 0 0
   [junit4]   2> 1924164 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[170]} 0 0
   [junit4]   2> 1924164 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[171]} 0 0
   [junit4]   2> 1924165 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[172]} 0 0
   [junit4]   2> 1924165 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[173]} 0 0
   [junit4]   2> 1924165 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[174]} 0 0
   [junit4]   2> 1924166 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[175]} 0 0
   [junit4]   2> 1924179 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[176]} 0 0
   [junit4]   2> 1924179 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[177]} 0 0
   [junit4]   2> 1924179 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[178]} 0 0
   [junit4]   2> 1924180 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[179]} 0 0
   [junit4]   2> 1924180 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[180]} 0 0
   [junit4]   2> 1924180 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[181]} 0 0
   [junit4]   2> 1924181 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[182]} 0 0
   [junit4]   2> 1924181 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[183]} 0 0
   [junit4]   2> 1924181 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[184]} 0 0
   [junit4]   2> 1924182 INFO  (qtp539172872-5138) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[185]} 0 0
   [junit4]   2> 1924182 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[186]} 0 0
   [junit4]   2> 1924182 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[187]} 0 0
   [junit4]   2> 1924183 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[188]} 0 0
   [junit4]   2> 1924183 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[189]} 0 0
   [junit4]   2> 1924183 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[190]} 0 0
   [junit4]   2> 1924184 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[191]} 0 0
   [junit4]   2> 1924184 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[192]} 0 0
   [junit4]   2> 1924184 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[193]} 0 0
   [junit4]   2> 1924185 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[194]} 0 0
   [junit4]   2> 1924185 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[195]} 0 0
   [junit4]   2> 1924185 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[196]} 0 0
   [junit4]   2> 1924186 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[197]} 0 0
   [junit4]   2> 1924186 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[198]} 0 0
   [junit4]   2> 1924186 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[199]} 0 0
   [junit4]   2> 1924207 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[200]} 0 0
   [junit4]   2> 1924207 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[201]} 0 0
   [junit4]   2> 1924208 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[202]} 0 0
   [junit4]   2> 1924208 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[203]} 0 0
   [junit4]   2> 1924208 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[204]} 0 0
   [junit4]   2> 1924209 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[205]} 0 0
   [junit4]   2> 1924209 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[206]} 0 0
   [junit4]   2> 1924209 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[207]} 0 0
   [junit4]   2> 1924211 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[208]} 0 0
   [junit4]   2> 1924211 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[209]} 0 0
   [junit4]   2> 1924211 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[210]} 0 0
   [junit4]   2> 1924211 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[211]} 0 0
   [junit4]   2> 1924212 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[212]} 0 0
   [junit4]   2> 1924212 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[213]} 0 0
   [junit4]   2> 1924212 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[214]} 0 0
   [junit4]   2> 1924213 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[215]} 0 0
   [junit4]   2> 1924213 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[216]} 0 0
   [junit4]   2> 1924213 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[217]} 0 0
   [junit4]   2> 1924213 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[218]} 0 0
   [junit4]   2> 1924214 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[219]} 0 0
   [junit4]   2> 1924214 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[220]} 0 0
   [junit4]   2> 1924214 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[221]} 0 0
   [junit4]   2> 1924214 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[222]} 0 0
   [junit4]   2> 1924215 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[223]} 0 0
   [junit4]   2> 1924215 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[224]} 0 0
   [junit4]   2> 1924215 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[225]} 0 0
   [junit4]   2> 1924216 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[226]} 0 0
   [junit4]   2> 1924216 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[227]} 0 0
   [junit4]   2> 1924216 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[228]} 0 0
   [junit4]   2> 1924217 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[229]} 0 0
   [junit4]   2> 1924217 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[230]} 0 0
   [junit4]   2> 1924217 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[231]} 0 0
   [junit4]   2> 1924218 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[232]} 0 0
   [junit4]   2> 1924218 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[233]} 0 0
   [junit4]   2> 1924219 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[234]} 0 0
   [junit4]   2> 1924240 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[235]} 0 0
   [junit4]   2> 1924240 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[236]} 0 0
   [junit4]   2> 1924240 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[237]} 0 0
   [junit4]   2> 1924241 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[238]} 0 0
   [junit4]   2> 1924241 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[239]} 0 0
   [junit4]   2> 1924241 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[240]} 0 0
   [junit4]   2> 1924242 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[241]} 0 0
   [junit4]   2> 1924242 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[242]} 0 0
   [junit4]   2> 1924242 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[243]} 0 0
   [junit4]   2> 1924243 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[244]} 0 0
   [junit4]   2> 1924243 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[245]} 0 0
   [junit4]   2> 1924244 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[246]} 0 0
   [junit4]   2> 1924244 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[247]} 0 0
   [junit4]   2> 1924244 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[248]} 0 0
   [junit4]   2> 1924245 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[249]} 0 0
   [junit4]   2> 1924245 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[250]} 0 0
   [junit4]   2> 1924245 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[251]} 0 0
   [junit4]   2> 1924245 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[252]} 0 0
   [junit4]   2> 1924246 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[253]} 0 0
   [junit4]   2> 1924246 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[254]} 0 0
   [junit4]   2> 1924246 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[255]} 0 0
   [junit4]   2> 1924247 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[256]} 0 0
   [junit4]   2> 1924247 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[257]} 0 0
   [junit4]   2> 1924256 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[258]} 0 0
   [junit4]   2> 1924256 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[259]} 0 0
   [junit4]   2> 1924257 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[260]} 0 0
   [junit4]   2> 1924257 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[261]} 0 0
   [junit4]   2> 1924257 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[262]} 0 0
   [junit4]   2> 1924258 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[263]} 0 0
   [junit4]   2> 1924258 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[264]} 0 0
   [junit4]   2> 1924258 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[265]} 0 0
   [junit4]   2> 1924259 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[266]} 0 0
   [junit4]   2> 1924259 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[267]} 0 0
   [junit4]   2> 1924272 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[268]} 0 0
   [junit4]   2> 1924272 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[269]} 0 0
   [junit4]   2> 1924272 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[270]} 0 0
   [junit4]   2> 1924273 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[271]} 0 0
   [junit4]   2> 1924273 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[272]} 0 0
   [junit4]   2> 1924273 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[273]} 0 0
   [junit4]   2> 1924274 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[274]} 0 0
   [junit4]   2> 1924274 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[275]} 0 0
   [junit4]   2> 1924274 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[276]} 0 0
   [junit4]   2> 1924275 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[277]} 0 0
   [junit4]   2> 1924275 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[278]} 0 0
   [junit4]   2> 1924275 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[279]} 0 0
   [junit4]   2> 1924275 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[280]} 0 0
   [junit4]   2> 1924276 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[281]} 0 0
   [junit4]   2> 1924276 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[282]} 0 0
   [junit4]   2> 1924276 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[283]} 0 0
   [junit4]   2> 1924276 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[284]} 0 0
   [junit4]   2> 1924277 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[285]} 0 0
   [junit4]   2> 1924277 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[286]} 0 0
   [junit4]   2> 1924277 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[287]} 0 0
   [junit4]   2> 1924278 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[288]} 0 0
   [junit4]   2> 1924278 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[289]} 0 0
   [junit4]   2> 1924278 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[290]} 0 0
   [junit4]   2> 1924278 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[291]} 0 0
   [junit4]   2> 1924279 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[292]} 0 0
   [junit4]   2> 1924279 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[293]} 0 0
   [junit4]   2> 1924279 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[294]} 0 0
   [junit4]   2> 1924279 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[295]} 0 0
   [junit4]   2> 1924280 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[296]} 0 0
   [junit4]   2> 1924288 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[297]} 0 0
   [junit4]   2> 1924289 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[298]} 0 0
   [junit4]   2> 1924290 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[299]} 0 0
   [junit4]   2> 1924290 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[300]} 0 0
   [junit4]   2> 1924291 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[301]} 0 0
   [junit4]   2> 1924291 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[302]} 0 0
   [junit4]   2> 1924291 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[303]} 0 0
   [junit4]   2> 1924292 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[304]} 0 0
   [junit4]   2> 1924304 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[305]} 0 0
   [junit4]   2> 1924305 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[306]} 0 0
   [junit4]   2> 1924305 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[307]} 0 0
   [junit4]   2> 1924305 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[308]} 0 0
   [junit4]   2> 1924306 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[309]} 0 0
   [junit4]   2> 1924306 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[310]} 0 0
   [junit4]   2> 1924307 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[311]} 0 0
   [junit4]   2> 1924307 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[312]} 0 0
   [junit4]   2> 1924307 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[313]} 0 0
   [junit4]   2> 1924308 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[314]} 0 0
   [junit4]   2> 1924316 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[315]} 0 8
   [junit4]   2> 1924317 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[316]} 0 0
   [junit4]   2> 1924317 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[317]} 0 0
   [junit4]   2> 1924317 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[318]} 0 0
   [junit4]   2> 1924318 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[319]} 0 0
   [junit4]   2> 1924318 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[320]} 0 0
   [junit4]   2> 1924318 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[321]} 0 0
   [junit4]   2> 1924318 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[322]} 0 0
   [junit4]   2> 1924319 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[323]} 0 0
   [junit4]   2> 1924319 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[324]} 0 0
   [junit4]   2> 1924319 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[325]} 0 0
   [junit4]   2> 1924320 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[326]} 0 0
   [junit4]   2> 1924320 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[327]} 0 0
   [junit4]   2> 1924320 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[328]} 0 0
   [junit4]   2> 1924320 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[329]} 0 0
   [junit4]   2> 1924321 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[330]} 0 0
   [junit4]   2> 1924321 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[331]} 0 0
   [junit4]   2> 1924321 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[332]} 0 0
   [junit4]   2> 1924322 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[333]} 0 0
   [junit4]   2> 1924322 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[334]} 0 0
   [junit4]   2> 1924323 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[335]} 0 0
   [junit4]   2> 1924323 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[336]} 0 0
   [junit4]   2> 1924323 INFO  (qtp539172872-5136) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[337]} 0 0
   [junit4]   2> 1924324 INFO  (qtp539172872-5137) [    x:collection1] o.a.s.u.p.LogUpdateProcesso

[...truncated too long message...]

(TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2bc761c9: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2235673a
   [junit4]   2> 2640169 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2640169 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@733cc63: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6358ef4f
   [junit4]   2> 2640194 INFO  (coreCloseExecutor-2995-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@12a57a4a
   [junit4]   2> 2640194 INFO  (coreCloseExecutor-2995-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=312834634
   [junit4]   2> 2640195 INFO  (coreCloseExecutor-2995-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3376cf61: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@5ab8b0f9
   [junit4]   2> 2640209 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@48eb6196{/solr,null,UNAVAILABLE}
   [junit4]   2> 2640209 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 2640210 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@11468d0f{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2640211 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1711618944
   [junit4]   2> 2640211 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2640211 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@50302ce2: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@5b8c2752
   [junit4]   2> 2640212 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2640212 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6f586d2a: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2235673a
   [junit4]   2> 2640213 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2640213 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3316de6: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6358ef4f
   [junit4]   2> 2640213 INFO  (coreCloseExecutor-2997-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6cd75570
   [junit4]   2> 2640213 INFO  (coreCloseExecutor-2997-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1826051440
   [junit4]   2> 2640214 INFO  (coreCloseExecutor-2997-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@229e1ae1: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@13bc9e7e
   [junit4]   2> 2640216 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7e65be9c{/solr,null,UNAVAILABLE}
   [junit4]   2> 2640216 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> 2640217 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@34bd21a3{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2640217 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=109938631
   [junit4]   2> 2640217 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2640217 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@42655f8e: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@12e9f47c
   [junit4]   2> 2640218 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2640218 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@31b93e5e: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2235673a
   [junit4]   2> 2640219 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2640219 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@198b506: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6358ef4f
   [junit4]   2> 2640220 INFO  (coreCloseExecutor-2999-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4b6806c6
   [junit4]   2> 2640220 INFO  (coreCloseExecutor-2999-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1265108678
   [junit4]   2> 2640220 INFO  (coreCloseExecutor-2999-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@58f917be: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@6ab07ad3
   [junit4]   2> 2640231 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@160e5b6{/solr,null,UNAVAILABLE}
   [junit4]   2> 2640231 INFO  (TEST-TestReplicationHandler.doTestDetails-seed#[D615EA42C65A9D66]) [    ] o.e.j.s.session Stopped scavenging
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_D615EA42C65A9D66-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {name=PostingsFormat(name=Direct), id=PostingsFormat(name=Direct), newname=PostingsFormat(name=Direct)}, docValues:{}, maxPointsInLeafNode=1580, maxMBSortInHeap=5.411833685128636, sim=RandomSimilarity(queryNorm=true): {}, locale=vi, timezone=Australia/Lindeman
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=145708688,total=527958016
   [junit4]   2> NOTE: All tests run in this JVM: [OverseerRolesTest, QueryElevationComponentTest, CollectionsAPIAsyncDistributedZkTest, DeleteInactiveReplicaTest, GraphQueryTest, TestNumericRangeQuery32, TestManagedSynonymGraphFilterFactory, ConnectionManagerTest, SolrCloudReportersTest, DistribCursorPagingTest, CollectionsAPISolrJTest, TestFastLRUCache, TestInfoStreamLogging, SOLR749Test, PathHierarchyTokenizerFactoryTest, TestManagedResourceStorage, TestReversedWildcardFilterFactory, TestUniqueKeyFieldResource, TestHdfsCloudBackupRestore, TestRawResponseWriter, TestReload, DistributedFacetExistsSmallTest, TestDocBasedVersionConstraints, MultiThreadedOCPTest, LoggingHandlerTest, BasicAuthIntegrationTest, PolyFieldTest, TestTlogReplica, TestSimpleTrackingShardHandler, CustomHighlightComponentTest, TestSortableTextField, SampleTest, TestCloudJSONFacetJoinDomain, BlockDirectoryTest, TestFunctionQuery, PrimUtilsTest, ActionThrottleTest, TestFieldTypeResource, DocValuesMissingTest, TestHttpShardHandlerFactory, TestStressRecovery, CSVRequestHandlerTest, TestSolrJ, TestStressVersions, TestReplicationHandler]
   [junit4] Completed [171/785 (1!)] on J1 in 717.67s, 16 tests, 1 failure, 2 skipped <<< FAILURES!

[...truncated 51700 lines...]