You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/02/06 06:32:23 UTC
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 2096 - Still
Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2096/
3 tests failed.
FAILED: org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh
Error Message:
Captured an uncaught exception in thread: Thread[id=17673, name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17673, name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
at org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
at org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
at org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
FAILED: junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager
Error Message:
The test or suite printed 16173 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
Stack Trace:
java.lang.AssertionError: The test or suite printed 16173 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
at org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:282)
at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
at java.base/java.lang.Thread.run(Thread.java:834)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.cdcr.CdcrBootstrapTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) >
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
>
at __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotNull(Assert.java:755)
at org.junit.Assert.assertNull(Assert.java:737)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
at jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
at java.base/java.lang.Thread.run(Thread.java:834)
Build Log:
[...truncated 2104 lines...]
[junit4] Suite: org.apache.lucene.search.TestSearcherManager
[junit4] 2> Feb 06, 2020 3:18:49 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-17208,5,TGRP-TestSearcherManager]
[junit4] 2> java.lang.RuntimeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
[junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] 2> at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
[junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
[junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
[junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
[junit4] 2> at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
[junit4] 2> at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
[junit4] 2> at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
[junit4] 2> at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
[junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
[junit4] 2> at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
[junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
[junit4] 2> at org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
[junit4] 2> at org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
[junit4] 2> at org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
[junit4] 2> at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
[junit4] 2> at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
[junit4] 2> at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
[junit4] 2> at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
[junit4] 2> at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
[junit4] 2> at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
[junit4] 2> at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
[junit4] 2> at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
[junit4] 2> at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
[junit4] 2> at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
[junit4] 2>
[junit4] 2> Feb 06, 2020 3:18:50 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-17205,5,TGRP-TestSearcherManager]
[junit4] 2> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
[junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
[junit4] 2> at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
[junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
[junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
[junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
[junit4] 2> at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
[junit4] 2> at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
[junit4] 2> at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
[junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
[junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] 2> at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
[junit4] 2> at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
[junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
[junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
[junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
[junit4] 2> at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
[junit4] 2>
[junit4] 2> Feb 06, 2020 3:18:50 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #6,5,TGRP-TestSearcherManager]
[junit4] 2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp: Too many open files
[junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
[junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp: Too many open files
[junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
[junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
[junit4] 2> at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
[junit4] 2> at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
[junit4] 2> at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
[junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
[junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] 2> at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
[junit4] 2> at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
[junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
[junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
[junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
[junit4] 2> at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
[junit4] 2>
[junit4] 2> Feb 06, 2020 3:18:50 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #4,5,TGRP-TestSearcherManager]
[junit4] 2> org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
[junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
[junit4] 2> at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
[junit4] 2> at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
[junit4] 2> at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
[junit4] 2> at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
[junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
[junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
[junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
[junit4] 2> at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
[junit4] 2> at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
[junit4] 2> at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
[junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
[junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] 2> at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
[junit4] 2> at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
[junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
[junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
[junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
[junit4] 2> at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
[junit4] 2> ... 3 more
[junit4] 2>
[junit4] 2> Feb 06, 2020 3:18:51 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #5,5,TGRP-TestSearcherManager]
[junit4] 2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
[junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
[junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
[junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
[junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
[junit4] 2> at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
[junit4] 2> at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
[junit4] 2> at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
[junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
[junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] 2> at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
[junit4] 2> at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
[junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
[junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
[junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
[junit4] 2> at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
[junit4] 2>
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSearcherManager -Dtests.method=testConcurrentIndexCloseSearchAndRefresh -Dtests.seed=CBAB78174F1B51A9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 17.7s J0 | TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17673, name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
[junit4] > Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
[junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] > at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
[junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
[junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
[junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
[junit4] > at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
[junit4] > at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
[junit4] > at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
[junit4] > at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
[junit4] > at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
[junit4] > at org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
[junit4] > at org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
[junit4] > at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
[junit4] > at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
[junit4] > at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
[junit4] > at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
[junit4] > at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
[junit4] > at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
[junit4] > at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
[junit4] > at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
[junit4] > at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
[junit4] > at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17688, name=Lucene Merge Thread #6, state=RUNNABLE, group=TGRP-TestSearcherManager]
[junit4] > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp: Too many open files
[junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
[junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp: Too many open files
[junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
[junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
[junit4] > at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
[junit4] > at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
[junit4] > at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
[junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] > at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
[junit4] > at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
[junit4] > at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17670, name=Thread-17205, state=RUNNABLE, group=TGRP-TestSearcherManager]
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
[junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
[junit4] > at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
[junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
[junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
[junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
[junit4] > at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
[junit4] > at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
[junit4] > at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
[junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] > at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
[junit4] > at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
[junit4] > at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17686, name=Lucene Merge Thread #4, state=RUNNABLE, group=TGRP-TestSearcherManager]
[junit4] > Caused by: org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
[junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
[junit4] > at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
[junit4] > at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
[junit4] > at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
[junit4] > at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
[junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
[junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
[junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
[junit4] > at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
[junit4] > at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
[junit4] > at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
[junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] > at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
[junit4] > at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
[junit4] > at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
[junit4] > ... 3 moreThrowable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17687, name=Lucene Merge Thread #5, state=RUNNABLE, group=TGRP-TestSearcherManager]
[junit4] > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
[junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
[junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
[junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
[junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
[junit4] > at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
[junit4] > at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
[junit4] > at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
[junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
[junit4] > at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
[junit4] > at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
[junit4] > at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene84), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@277d98a5), locale=en-US, timezone=Pacific/Tarawa
[junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=4,threads=1,free=244326912,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [TestSameScoresWithThreads, TestLucene84PostingsFormat, TestRamUsageEstimator, TestLatLonMultiLineShapeQueries, TestEarlyTermination, TestSimpleFSLockFactory, TestMatchesIterator, TestStressIndexing, TestMultiFields, TestIndexCommit, TestDisjunctionScoreBlockBoundaryPropagator, TestMixedCodecs, TestLucene50CompoundFormat, TestField, TestTotalHits, TestIndexedDISI, TestSegmentCacheables, TestRegExp, TestDocIdsWriter, TestFieldReuse, TestIntBlockPool, TestIndexWriterThreadsToSegments, TestReaderPool, TestNoMergePolicy, TestPerFieldPostingsFormat, MultiCollectorTest, TestComplexExplanationsOfNonMatches, TestLogMergePolicy, TestTermStates, TestPendingDeletes, TestFeatureDoubleValues, TestMultiset, TestSortRescorer, TestRollingUpdates, TestSimilarityBase, TestTermVectorsWriter, TestXYRectangle2D, TestAddIndexes, TestTransactions, TestSpansEnum, TestVirtualMethod, TestIndexManyDocuments, TestGeoUtils, TestLatLonPointShapeQueries, TestConstantScoreQuery, TestWordlistLoader, TestIntsRef, TestIndexSorting, TestMaxScoreSumPropagator, TestNeverDelete, TestSegmentTermEnum, TestLucene60FieldInfoFormat, TestXYLineShapeQueries, TestFieldValueQuery, TestPrefixCodedTerms, TestBM25Similarity, TestAxiomaticF1LOG, TestFixedBitDocIdSet, TestUniqueTermCount, TestFilterMergePolicy, TestTimeLimitingCollector, TestPhraseQuery, TestMaxPosition, TestSimpleExplanationsWithFillerDocs, TestSentinelIntSet, TestDocValuesRewriteMethod, TestDaciukMihovAutomatonBuilder, TestNRTReaderCleanup, TestCustomNorms, TestSnapshotDeletionPolicy, TestDuelingCodecs, TestStringHelper, TestIntRange, TestAllFilesHaveChecksumFooter, TestCharsRef, TestPriorityQueue, TestLevenshteinAutomata, TestFuzzyQuery, TestXYShape, TestBinaryTerms, TestNumericUtils, TestPrefixRandom, TestAutomatonQuery, TestTimSorterWorstCase, TestIndexWriterOnVMError, TestXYMultiPolygonShapeQueries, TestDirectMonotonic, TestLatLonPoint, TestLongValuesSource, TestNativeFSLockFactory, TestIntRangeFieldQueries, TestSortedSetSortField, TestLucene50StoredFieldsFormatMergeInstance, TestLucene80NormsFormatMergeInstance, Test2BPostingsBytes, Test2BSortedDocValuesFixedSorted, Test2BSortedDocValuesOrds, Test2BTerms, Test4GBStoredFields, TestBagOfPositions, TestBagOfPostings, TestBinaryDocValuesUpdates, TestCodecHoldsOpenFiles, TestCodecs, TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash, TestCrashCausesCorruptIndex, TestDemoParallelLeafReader, TestExitableDirectoryReader, TestFieldInfos, TestFieldInvertState, TestFilterCodecReader, TestFilterDirectoryReader, TestFilterLeafReader, TestForTooMuchCloning, TestForceMergeForever, TestFrozenBufferedUpdates, TestIndexWriterCommit, TestIndexWriterConfig, TestIndexWriterDelete, TestIndexWriterDeleteByQuery, TestIndexWriterExceptions, TestIndexWriterOnDiskFull, TestOmitNorms, TestOmitPositions, TestOmitTf, TestOneMergeWrappingMergePolicy, TestOrdinalMap, TestParallelCompositeReader, TestParallelLeafReader, TestParallelReaderEmptyIndex, TestParallelTermEnum, TestPayloads, TestPayloadsOnVectors, TestPersistentSnapshotDeletionPolicy, TestPointValues, TestPostingsOffsets, TestSegmentMerger, TestSegmentReader, TestSegmentTermDocs, TestSortedSetDocValues, TestStressAdvance, TestStressDeletes, TestSumDocFreq, TestSwappedIndexFiles, TestTermsEnum2, TestThreadedForceMerge, TestTieredMergePolicy, TestTragicIndexWriterDeadlock, TestTransactionRollback, TestBlockMaxConjunction, TestBoolean2, TestBoolean2ScorerSupplier, TestBooleanMinShouldMatch, TestBooleanOr, TestBooleanQuery, TestBooleanQueryVisitSubscorers, TestBooleanRewrites, TestBooleanScorer, TestBoostQuery, TestCachingCollector, TestComplexExplanations, TestCustomSearcherSort, TestDateSort, TestDisjunctionMaxQuery, TestLatLonPointQueries, TestLiveFieldValues, TestLongRangeFieldQueries, TestSearcherManager]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSearcherManager -Dtests.seed=CBAB78174F1B51A9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | TestSearcherManager (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: The test or suite printed 16173 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
[junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
[junit4] Completed [546/552 (1!)] on J0 in 355.57s, 11 tests, 1 failure, 1 error <<< FAILURES!
[...truncated 12649 lines...]
[junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBootstrapTest
[junit4] 2> 5121336 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/data-dir-186-001
[junit4] 2> 5121337 WARN (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11
[junit4] 2> 5121337 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 5121337 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0)
[junit4] 2> 5121338 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 5121341 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithMultipleReplicas
[junit4] 2> 5121341 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001
[junit4] 2> 5121341 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 5121442 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer start zk server on port:36855
[junit4] 2> 5121442 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:36855
[junit4] 2> 5121442 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36855
[junit4] 2> 5121442 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36855
[junit4] 2> 5121443 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5121452 INFO (zkConnectionManagerCallback-12879-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5121452 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5121454 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5121455 INFO (zkConnectionManagerCallback-12881-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5121455 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5121456 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5121457 INFO (zkConnectionManagerCallback-12883-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5121457 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
[junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
[junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@867eda0{/solr,null,AVAILABLE}
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d1dd263{/solr,null,AVAILABLE}
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ecf5186{/solr,null,AVAILABLE}
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3473a5ca{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:41191}
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.Server Started @5121593ms
[junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41191}
[junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@77d77ae2{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:37757}
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.Server Started @5121593ms
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37757}
[junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-02-06T05:48:19.821958Z
[junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-02-06T05:48:19.821957Z
[junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@16db638b{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:43461}
[junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.Server Started @5121594ms
[junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43461}
[junit4] 2> 5121567 ERROR (jetty-launcher-12884-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
[junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-02-06T05:48:19.822974Z
[junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5121569 INFO (zkConnectionManagerCallback-12886-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5121569 INFO (zkConnectionManagerCallback-12888-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5121569 INFO (zkConnectionManagerCallback-12890-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 5122191 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@9c25133[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@9c25133[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3dc1efed[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3dc1efed[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5122224 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
[junit4] 2> 5122229 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5122230 INFO (zkConnectionManagerCallback-12899-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5122230 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5122332 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5122333 INFO (zkConnectionManagerCallback-12901-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5122333 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5122450 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37757_solr
[junit4] 2> 5122451 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Overseer (id=73197244948086791-127.0.0.1:37757_solr-n_0000000000) starting
[junit4] 2> 5122454 INFO (OverseerStateUpdate-73197244948086791-127.0.0.1:37757_solr-n_0000000000) [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:37757_solr
[junit4] 2> 5122464 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37757_solr
[junit4] 2> 5122467 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 5122467 WARN (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 5122475 INFO (zkCallback-12900-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5122500 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 5122523 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37757.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37757.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37757.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5122533 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node1/.
[junit4] 2> 5122737 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 5122738 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2d6e218d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5122739 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2d6e218d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1ed7f896[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1ed7f896[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5122743 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
[junit4] 2> 5122744 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5122746 INFO (zkConnectionManagerCallback-12911-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5122746 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5122848 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5122852 INFO (zkConnectionManagerCallback-12913-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5122852 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5122864 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5122868 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:43461_solr as DOWN
[junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43461_solr
[junit4] 2> 5122870 INFO (zkCallback-12900-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5122870 INFO (zkCallback-12912-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5122871 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 5122871 WARN (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 5122894 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 5122918 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43461.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5122929 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43461.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5122930 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43461.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5122931 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node3/.
[junit4] 2> 5123682 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@32f87da4[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@32f87da4[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@32434175[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@32434175[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5123687 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
[junit4] 2> 5123688 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5123693 INFO (zkConnectionManagerCallback-12923-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5123693 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5123802 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5123803 INFO (zkConnectionManagerCallback-12925-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5123803 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5123810 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5123813 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:41191_solr as DOWN
[junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41191_solr
[junit4] 2> 5123815 INFO (zkCallback-12912-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5123815 INFO (zkCallback-12900-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5123817 INFO (zkCallback-12924-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5123817 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 5123818 WARN (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 5123839 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 5123860 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41191.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41191.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41191.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
[junit4] 2> 5123869 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node2/.
[junit4] 2> 5123900 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=3
[junit4] 2> 5123901 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5123902 INFO (zkConnectionManagerCallback-12934-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5123903 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5123904 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 5123905 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36855/solr ready
[junit4] 1> Target zkHost = 127.0.0.1:36855/solr
[junit4] 2> 5123905 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-source-001
[junit4] 2> 5123906 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 5124006 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer start zk server on port:40811
[junit4] 2> 5124006 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:40811
[junit4] 2> 5124006 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40811
[junit4] 2> 5124006 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40811
[junit4] 2> 5124009 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5124011 INFO (zkConnectionManagerCallback-12936-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5124011 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5124013 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5124014 INFO (zkConnectionManagerCallback-12938-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5124014 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5124015 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 5124015 INFO (zkConnectionManagerCallback-12940-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 5124015 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-3) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-2) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-1) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
[junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
[...truncated too long message...]
p(Native Method)
[junit4] 2> app//com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2>
[junit4] 2> 5279049 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> NOTE: test params are: codec=SimpleText, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@8a39653), locale=en-SD, timezone=VST
[junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=4,threads=1,free=386818192,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [AuditLoggerPluginTest, CdcrUpdateLogTest, SolrCloudExampleTest, ZookeeperStatusHandlerTest, UUIDFieldTest, NodeLostTriggerTest, TestOverriddenPrefixQueryForCustomFieldType, TestRandomFaceting, TestIBSimilarityFactory, TestUtils, UpdateLogTest, TestConfigOverlay, PeerSyncWithBufferUpdatesTest, TestHashQParserPlugin, TestLegacyBM25SimilarityFactory, CachingDirectoryFactoryTest, SortByFunctionTest, TestZkChroot, IgnoreLargeDocumentProcessorFactoryTest, TestPHPSerializedResponseWriter, TestInPlaceUpdatesRequiredField, TestJettySolrRunner, TestSerializedLuceneMatchVersion, DOMUtilTest, CoreAdminHandlerTest, TestManagedResource, TestSolrIndexConfig, BJQParserTest, TestSimLargeCluster, TestStressThreadBackup, AsyncCallRequestStatusResponseTest, TestImplicitCoreProperties, DistributedFacetPivotLongTailTest, WrapperMergePolicyFactoryTest, TestConfigSetsAPIExclusivity, TestRTimerTree, SolrJmxReporterCloudTest, QueryElevationComponentTest, NotRequiredUniqueKeyTest, ChaosMonkeyNothingIsSafeTest, MBeansHandlerTest, TestHighlightDedupGrouping, ZkShardTermsTest, TestJmxIntegration, CurrencyRangeFacetCloudTest, ShowFileRequestHandlerTest, TestRTGBase, TestCloudSearcherWarming, TestRawTransformer, SignificantTermsQParserPluginTest, MinimalSchemaTest, TestGeoJSONResponseWriter, AutoAddReplicasIntegrationTest, TestChildDocTransformer, TestStandardQParsers, TestNestedDocsSort, TestReRankQParserPlugin, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest, SolrCoreCheckLockOnStartupTest, TestSSLRandomization, TestLegacyNumericRangeQueryBuilder, HdfsBasicDistributedZkTest, OverseerSolrResponseTest, TestPKIAuthenticationPlugin, DistributedFacetPivotWhiteBoxTest, TermVectorComponentDistributedTest, Tagger2Test, PeerSyncWithLeaderTest, TestLRUStatsCacheCloud, ZkStateReaderTest, LeaderFailoverAfterPartitionTest, MultiSolrCloudTestCaseTest, LeaderElectionContextKeyTest, TestTlogReplica, SearchRateTriggerIntegrationTest, TestCollationField, TestHighFrequencyDictionaryFactory, NodeAddedTriggerIntegrationTest, TestConfigReload, TestSimPolicyCloud, DistributedQueryComponentOptimizationTest, SolrCmdDistributorTest, TestDynamicLoading, V2StandaloneTest, TestFiltering, CollectionPropsTest, TestSimComputePlanAction, CloudExitableDirectoryReaderTest, PrimUtilsTest, HttpPartitionOnCommitTest, ForceLeaderWithTlogReplicasTest, TestSchemaManager, TestMultiValuedNumericRangeQuery, TestClassicSimilarityFactory, TestBinaryResponseWriter, DefaultValueUpdateProcessorTest, TestSolrCloudWithKerberosAlt, CopyFieldTest, HdfsNNFailoverTest, TestManagedSynonymFilterFactory, HdfsTlogReplayBufferedWhileIndexingTest, AdminHandlersProxyTest, TestSolrTestCaseJ4, TestSolrConfigHandler, QueryParsingTest, TestCloudJSONFacetJoinDomain, TestCollectionAPIs, TestSolrJ, TestStressInPlaceUpdates, HttpTriggerListenerTest, SpellCheckCollatorWithCollapseTest, OverseerCollectionConfigSetProcessorTest, DistributedFacetSimpleRefinementLongTailTest, ScheduledMaintenanceTriggerTest, DistributedSuggestComponentTest, TestPerFieldSimilarityWithDefaultOverride, TestFieldTypeCollectionResource, TestExtendedDismaxParser, TestDelegationWithHadoopAuth, CloudMLTQParserTest, MetricsHistoryWithAuthIntegrationTest, ClassificationUpdateProcessorIntegrationTest, TestLocalFSCloudBackupRestore, TestSizeLimitedDistributedMap, XMLAtomicUpdateMultivalueTest, ShardSplitTest, MoveReplicaHDFSFailoverTest, TestSurroundQueryParser, CoreMergeIndexesAdminHandlerTest, BasicAuthStandaloneTest, TestSolrCachePerf, TestDistributedSearch, StressHdfsTest, TestBulkSchemaConcurrent, TestDownShardTolerantSearch, HdfsRestartWhileUpdatingTest, TestBadConfig, SolrInfoBeanTest, XmlInterpolationTest, TestDistributedStatsComponentCardinality, LeaderVoteWaitTimeoutTest, SearchHandlerTest, TestMacros, BasicDistributedZk2Test, CollectionsAPIAsyncDistributedZkTest, TestExactStatsCacheCloud, TestLeaderElectionZkExpiry, TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplica, TestPullReplicaErrorHandling, TestRandomRequestDistribution, TestRebalanceLeaders, TestTolerantUpdateProcessorRandomCloud, TestUtilizeNode, TestWaitForStateWithJettyShutdowns, TriLevelCompositeIdRoutingTest, UnloadDistributedZkTest, VMParamsZkACLAndCredentialsProvidersTest, ZkCLITest, ConcurrentDeleteAndCreateCollectionTest, TestCollectionsAPIViaSolrCloudCluster, AutoScalingHandlerTest, ExecutePlanActionTest, IndexSizeTriggerSizeEstimationTest, MetricTriggerIntegrationTest, SystemLogListenerTest, TestPolicyCloud, TriggerCooldownIntegrationTest, TriggerSetPropertiesIntegrationTest, TestSimDistributedQueue, TestSimExecutePlanAction, TestSimExtremeIndexing, TestSimTriggerIntegration, TestSimUtils, TestSnapshotCloudManager, BaseCdcrDistributedZkTest, CdcrBidirectionalTest, CdcrBootstrapTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=CdcrBootstrapTest -Dtests.seed=AE624A5368110313 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-SD -Dtests.timezone=VST -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | CdcrBootstrapTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
[junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
[junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
[junit4] > at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
[junit4] > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
[junit4] > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
[junit4] > at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
[junit4] > at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[junit4] > at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
[junit4] > expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
[junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
[junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
[junit4] > at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
[junit4] > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
[junit4] > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
[junit4] > at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
[junit4] > at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[junit4] > at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
[junit4] > >
[junit4] > at __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
[junit4] > at jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
[junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:566)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
[junit4] Completed [572/900 (1!)] on J1 in 157.86s, 4 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 53511 lines...]
Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 2096 - Still
Unstable
Posted by Erick Erickson <er...@gmail.com>.
Adrien:
We were seeing this very rarely after I changed to using async logging. There was a time when some logging tests would have log messages queued up for delivery in a test with lots and lots and lots of output that would somehow bleed over into an ensuing test and trigger this condition.
I collect these and occasionally see if I can see any pattern, there’s one from the same test last November.
I have no evidence one way or the other whether this is a result of the async logging changes or your suspicion about crazy index-writer configs, but wanted to mention in case you find it relevant.
Problem is, these happen maybe once a month so it makes it really hard to work on. There’s code in the logger that _should_ flush all the messages so perhaps these are legitimate issues after all. But I’ve been unable to reproduce reliably.
Best,
Erick
> On Feb 7, 2020, at 4:46 AM, Adrien Grand <jp...@gmail.com> wrote:
>
> I think it has little to do with the recent changes, but is more a combination of crazy index-writer configs (low maxBufferedDocs, high maxSegmentsPerTier) and concurrency.
>
> On Thu, Feb 6, 2020 at 5:27 PM Adrien Grand <jp...@gmail.com> wrote:
> I'm looking into it.
>
> On Thu, Feb 6, 2020 at 7:36 AM Apache Jenkins Server <je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2096/
>
> 3 tests failed.
> FAILED: org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=17673, name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17673, name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
> Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> at org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> at org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> at org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
>
>
> FAILED: junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager
>
> Error Message:
> The test or suite printed 16173 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
>
> Stack Trace:
> java.lang.AssertionError: The test or suite printed 16173 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> at org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:282)
> at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
>
> FAILED: junit.framework.TestSuite.org.apache.solr.cloud.cdcr.CdcrBootstrapTest
>
> Error Message:
> ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) >
>
> Stack Trace:
> java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
> at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
> expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
> at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
> >
> at __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.failNotNull(Assert.java:755)
> at org.junit.Assert.assertNull(Assert.java:737)
> at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
> at jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
> at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
>
>
>
> Build Log:
> [...truncated 2104 lines...]
> [junit4] Suite: org.apache.lucene.search.TestSearcherManager
> [junit4] 2> Feb 06, 2020 3:18:49 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-17208,5,TGRP-TestSearcherManager]
> [junit4] 2> java.lang.RuntimeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
> [junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> [junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
> [junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> [junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> [junit4] 2> at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> [junit4] 2> at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> [junit4] 2> at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> [junit4] 2> at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> [junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> [junit4] 2> at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> [junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> [junit4] 2> at org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> [junit4] 2> at org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> [junit4] 2> at org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> [junit4] 2> at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> [junit4] 2> at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> [junit4] 2> at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> [junit4] 2> at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> [junit4] 2> at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> [junit4] 2> at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> [junit4] 2> at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> [junit4] 2> at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> [junit4] 2> at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-17205,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
> [junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
> [junit4] 2> at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
> [junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
> [junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #6,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp: Too many open files
> [junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp: Too many open files
> [junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #4,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
> [junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
> [junit4] 2> at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
> [junit4] 2> at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
> [junit4] 2> at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
> [junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> ... 3 more
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:51 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #5,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
> [junit4] 2> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
> [junit4] 2> at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSearcherManager -Dtests.method=testConcurrentIndexCloseSearchAndRefresh -Dtests.seed=CBAB78174F1B51A9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true -Dtests.file.encoding=UTF-8
> [junit4] ERROR 17.7s J0 | TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh <<<
> [junit4] > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17673, name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
> [junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> [junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm: Too many open files
> [junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> [junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> [junit4] > at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> [junit4] > at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> [junit4] > at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> [junit4] > at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> [junit4] > at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> [junit4] > at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> [junit4] > at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> [junit4] > at org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> [junit4] > at org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> [junit4] > at org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> [junit4] > at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> [junit4] > at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> [junit4] > at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> [junit4] > at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> [junit4] > at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> [junit4] > at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> [junit4] > at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> [junit4] > at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> [junit4] > at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> [junit4] > at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17688, name=Lucene Merge Thread #6, state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp: Too many open files
> [junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp: Too many open files
> [junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17670, name=Thread-17205, state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
> [junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
> [junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
> [junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
> [junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
> [junit4] > at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
> [junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
> [junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17686, name=Lucene Merge Thread #4, state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by: org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
> [junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
> [junit4] > at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
> [junit4] > at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
> [junit4] > at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
> [junit4] > at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
> [junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
> [junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
> [junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > ... 3 moreThrowable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17687, name=Lucene Merge Thread #5, state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
> [junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by: java.nio.file.FileSystemException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp: Too many open files
> [junit4] > at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2> NOTE: test params are: codec=Asserting(Lucene84), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@277d98a5), locale=en-US, timezone=Pacific/Tarawa
> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=4,threads=1,free=244326912,total=536870912
> [junit4] 2> NOTE: All tests run in this JVM: [TestSameScoresWithThreads, TestLucene84PostingsFormat, TestRamUsageEstimator, TestLatLonMultiLineShapeQueries, TestEarlyTermination, TestSimpleFSLockFactory, TestMatchesIterator, TestStressIndexing, TestMultiFields, TestIndexCommit, TestDisjunctionScoreBlockBoundaryPropagator, TestMixedCodecs, TestLucene50CompoundFormat, TestField, TestTotalHits, TestIndexedDISI, TestSegmentCacheables, TestRegExp, TestDocIdsWriter, TestFieldReuse, TestIntBlockPool, TestIndexWriterThreadsToSegments, TestReaderPool, TestNoMergePolicy, TestPerFieldPostingsFormat, MultiCollectorTest, TestComplexExplanationsOfNonMatches, TestLogMergePolicy, TestTermStates, TestPendingDeletes, TestFeatureDoubleValues, TestMultiset, TestSortRescorer, TestRollingUpdates, TestSimilarityBase, TestTermVectorsWriter, TestXYRectangle2D, TestAddIndexes, TestTransactions, TestSpansEnum, TestVirtualMethod, TestIndexManyDocuments, TestGeoUtils, TestLatLonPointShapeQueries, TestConstantScoreQuery, TestWordlistLoader, TestIntsRef, TestIndexSorting, TestMaxScoreSumPropagator, TestNeverDelete, TestSegmentTermEnum, TestLucene60FieldInfoFormat, TestXYLineShapeQueries, TestFieldValueQuery, TestPrefixCodedTerms, TestBM25Similarity, TestAxiomaticF1LOG, TestFixedBitDocIdSet, TestUniqueTermCount, TestFilterMergePolicy, TestTimeLimitingCollector, TestPhraseQuery, TestMaxPosition, TestSimpleExplanationsWithFillerDocs, TestSentinelIntSet, TestDocValuesRewriteMethod, TestDaciukMihovAutomatonBuilder, TestNRTReaderCleanup, TestCustomNorms, TestSnapshotDeletionPolicy, TestDuelingCodecs, TestStringHelper, TestIntRange, TestAllFilesHaveChecksumFooter, TestCharsRef, TestPriorityQueue, TestLevenshteinAutomata, TestFuzzyQuery, TestXYShape, TestBinaryTerms, TestNumericUtils, TestPrefixRandom, TestAutomatonQuery, TestTimSorterWorstCase, TestIndexWriterOnVMError, TestXYMultiPolygonShapeQueries, TestDirectMonotonic, TestLatLonPoint, TestLongValuesSource, TestNativeFSLockFactory, TestIntRangeFieldQueries, TestSortedSetSortField, TestLucene50StoredFieldsFormatMergeInstance, TestLucene80NormsFormatMergeInstance, Test2BPostingsBytes, Test2BSortedDocValuesFixedSorted, Test2BSortedDocValuesOrds, Test2BTerms, Test4GBStoredFields, TestBagOfPositions, TestBagOfPostings, TestBinaryDocValuesUpdates, TestCodecHoldsOpenFiles, TestCodecs, TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash, TestCrashCausesCorruptIndex, TestDemoParallelLeafReader, TestExitableDirectoryReader, TestFieldInfos, TestFieldInvertState, TestFilterCodecReader, TestFilterDirectoryReader, TestFilterLeafReader, TestForTooMuchCloning, TestForceMergeForever, TestFrozenBufferedUpdates, TestIndexWriterCommit, TestIndexWriterConfig, TestIndexWriterDelete, TestIndexWriterDeleteByQuery, TestIndexWriterExceptions, TestIndexWriterOnDiskFull, TestOmitNorms, TestOmitPositions, TestOmitTf, TestOneMergeWrappingMergePolicy, TestOrdinalMap, TestParallelCompositeReader, TestParallelLeafReader, TestParallelReaderEmptyIndex, TestParallelTermEnum, TestPayloads, TestPayloadsOnVectors, TestPersistentSnapshotDeletionPolicy, TestPointValues, TestPostingsOffsets, TestSegmentMerger, TestSegmentReader, TestSegmentTermDocs, TestSortedSetDocValues, TestStressAdvance, TestStressDeletes, TestSumDocFreq, TestSwappedIndexFiles, TestTermsEnum2, TestThreadedForceMerge, TestTieredMergePolicy, TestTragicIndexWriterDeadlock, TestTransactionRollback, TestBlockMaxConjunction, TestBoolean2, TestBoolean2ScorerSupplier, TestBooleanMinShouldMatch, TestBooleanOr, TestBooleanQuery, TestBooleanQueryVisitSubscorers, TestBooleanRewrites, TestBooleanScorer, TestBoostQuery, TestCachingCollector, TestComplexExplanations, TestCustomSearcherSort, TestDateSort, TestDisjunctionMaxQuery, TestLatLonPointQueries, TestLiveFieldValues, TestLongRangeFieldQueries, TestSearcherManager]
> [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSearcherManager -Dtests.seed=CBAB78174F1B51A9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true -Dtests.file.encoding=UTF-8
> [junit4] ERROR 0.00s J0 | TestSearcherManager (suite) <<<
> [junit4] > Throwable #1: java.lang.AssertionError: The test or suite printed 16173 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
> [junit4] > at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] Completed [546/552 (1!)] on J0 in 355.57s, 11 tests, 1 failure, 1 error <<< FAILURES!
>
> [...truncated 12649 lines...]
> [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBootstrapTest
> [junit4] 2> 5121336 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/data-dir-186-001
> [junit4] 2> 5121337 WARN (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11
> [junit4] 2> 5121337 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
> [junit4] 2> 5121337 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0)
> [junit4] 2> 5121338 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
> [junit4] 2> 5121341 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithMultipleReplicas
> [junit4] 2> 5121341 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001
> [junit4] 2> 5121341 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
> [junit4] 2> 5121442 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer start zk server on port:36855
> [junit4] 2> 5121442 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:36855
> [junit4] 2> 5121442 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36855
> [junit4] 2> 5121442 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36855
> [junit4] 2> 5121443 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121452 INFO (zkConnectionManagerCallback-12879-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121452 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121454 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121455 INFO (zkConnectionManagerCallback-12881-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121455 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121456 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121457 INFO (zkConnectionManagerCallback-12883-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121457 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@867eda0{/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.session node0 Scavenging every 660000ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d1dd263{/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ecf5186{/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3473a5ca{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:41191}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ] o.e.j.s.Server Started @5121593ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41191}
> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@77d77ae2{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:37757}
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.e.j.s.Server Started @5121593ms
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37757}
> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-02-06T05:48:19.821958Z
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-02-06T05:48:19.821957Z
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@16db638b{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:43461}
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.e.j.s.Server Started @5121594ms
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43461}
> [junit4] 2> 5121567 ERROR (jetty-launcher-12884-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-02-06T05:48:19.822974Z
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121569 INFO (zkConnectionManagerCallback-12886-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121569 INFO (zkConnectionManagerCallback-12888-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121569 INFO (zkConnectionManagerCallback-12890-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5122191 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@9c25133[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@9c25133[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3dc1efed[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3dc1efed[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122224 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5122229 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5122230 INFO (zkConnectionManagerCallback-12899-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122230 INFO (jetty-launcher-12884-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5122332 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5122333 INFO (zkConnectionManagerCallback-12901-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122333 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5122450 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37757_solr
> [junit4] 2> 5122451 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Overseer (id=73197244948086791-127.0.0.1:37757_solr-n_0000000000) starting
> [junit4] 2> 5122454 INFO (OverseerStateUpdate-73197244948086791-127.0.0.1:37757_solr-n_0000000000) [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:37757_solr
> [junit4] 2> 5122464 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37757_solr
> [junit4] 2> 5122467 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
> [junit4] 2> 5122467 WARN (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5122475 INFO (zkCallback-12900-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
> [junit4] 2> 5122500 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
> [junit4] 2> 5122523 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37757.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37757.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37757.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122533 INFO (jetty-launcher-12884-thread-1) [n:127.0.0.1:37757_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node1/.
> [junit4] 2> 5122737 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
> [junit4] 2> 5122738 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2d6e218d[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122739 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2d6e218d[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1ed7f896[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1ed7f896[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122743 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5122744 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5122746 INFO (zkConnectionManagerCallback-12911-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122746 INFO (jetty-launcher-12884-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5122848 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5122852 INFO (zkConnectionManagerCallback-12913-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122852 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5122864 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
> [junit4] 2> 5122868 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:43461_solr as DOWN
> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43461_solr
> [junit4] 2> 5122870 INFO (zkCallback-12900-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
> [junit4] 2> 5122870 INFO (zkCallback-12912-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
> [junit4] 2> 5122871 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
> [junit4] 2> 5122871 WARN (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5122894 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
> [junit4] 2> 5122918 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43461.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122929 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43461.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122930 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43461.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122931 INFO (jetty-launcher-12884-thread-3) [n:127.0.0.1:43461_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node3/.
> [junit4] 2> 5123682 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@32f87da4[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@32f87da4[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@32434175[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@32434175[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123687 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5123688 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5123693 INFO (zkConnectionManagerCallback-12923-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123693 INFO (jetty-launcher-12884-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5123802 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5123803 INFO (zkConnectionManagerCallback-12925-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123803 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5123810 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
> [junit4] 2> 5123813 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:41191_solr as DOWN
> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41191_solr
> [junit4] 2> 5123815 INFO (zkCallback-12912-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123815 INFO (zkCallback-12900-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123817 INFO (zkCallback-12924-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123817 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
> [junit4] 2> 5123818 WARN (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5123839 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
> [junit4] 2> 5123860 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41191.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41191.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41191.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123869 INFO (jetty-launcher-12884-thread-2) [n:127.0.0.1:41191_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node2/.
> [junit4] 2> 5123900 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=3
> [junit4] 2> 5123901 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5123902 INFO (zkConnectionManagerCallback-12934-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123903 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5123904 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
> [junit4] 2> 5123905 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36855/solr ready
> [junit4] 1> Target zkHost = 127.0.0.1:36855/solr
> [junit4] 2> 5123905 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-source-001
> [junit4] 2> 5123906 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
> [junit4] 2> 5124006 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer start zk server on port:40811
> [junit4] 2> 5124006 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:40811
> [junit4] 2> 5124006 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40811
> [junit4] 2> 5124006 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40811
> [junit4] 2> 5124009 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5124011 INFO (zkConnectionManagerCallback-12936-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124011 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124013 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5124014 INFO (zkConnectionManagerCallback-12938-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124014 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124015 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5124015 INFO (zkConnectionManagerCallback-12940-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124015 INFO (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-3) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-2) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-1) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>
>
> [...truncated too long message...]
>
> p(Native Method)
> [junit4] 2> app//com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
> [junit4] 2>
> [junit4] 2> 5279049 INFO (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
> [junit4] 2> NOTE: test params are: codec=SimpleText, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@8a39653), locale=en-SD, timezone=VST
> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=4,threads=1,free=386818192,total=536870912
> [junit4] 2> NOTE: All tests run in this JVM: [AuditLoggerPluginTest, CdcrUpdateLogTest, SolrCloudExampleTest, ZookeeperStatusHandlerTest, UUIDFieldTest, NodeLostTriggerTest, TestOverriddenPrefixQueryForCustomFieldType, TestRandomFaceting, TestIBSimilarityFactory, TestUtils, UpdateLogTest, TestConfigOverlay, PeerSyncWithBufferUpdatesTest, TestHashQParserPlugin, TestLegacyBM25SimilarityFactory, CachingDirectoryFactoryTest, SortByFunctionTest, TestZkChroot, IgnoreLargeDocumentProcessorFactoryTest, TestPHPSerializedResponseWriter, TestInPlaceUpdatesRequiredField, TestJettySolrRunner, TestSerializedLuceneMatchVersion, DOMUtilTest, CoreAdminHandlerTest, TestManagedResource, TestSolrIndexConfig, BJQParserTest, TestSimLargeCluster, TestStressThreadBackup, AsyncCallRequestStatusResponseTest, TestImplicitCoreProperties, DistributedFacetPivotLongTailTest, WrapperMergePolicyFactoryTest, TestConfigSetsAPIExclusivity, TestRTimerTree, SolrJmxReporterCloudTest, QueryElevationComponentTest, NotRequiredUniqueKeyTest, ChaosMonkeyNothingIsSafeTest, MBeansHandlerTest, TestHighlightDedupGrouping, ZkShardTermsTest, TestJmxIntegration, CurrencyRangeFacetCloudTest, ShowFileRequestHandlerTest, TestRTGBase, TestCloudSearcherWarming, TestRawTransformer, SignificantTermsQParserPluginTest, MinimalSchemaTest, TestGeoJSONResponseWriter, AutoAddReplicasIntegrationTest, TestChildDocTransformer, TestStandardQParsers, TestNestedDocsSort, TestReRankQParserPlugin, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest, SolrCoreCheckLockOnStartupTest, TestSSLRandomization, TestLegacyNumericRangeQueryBuilder, HdfsBasicDistributedZkTest, OverseerSolrResponseTest, TestPKIAuthenticationPlugin, DistributedFacetPivotWhiteBoxTest, TermVectorComponentDistributedTest, Tagger2Test, PeerSyncWithLeaderTest, TestLRUStatsCacheCloud, ZkStateReaderTest, LeaderFailoverAfterPartitionTest, MultiSolrCloudTestCaseTest, LeaderElectionContextKeyTest, TestTlogReplica, SearchRateTriggerIntegrationTest, TestCollationField, TestHighFrequencyDictionaryFactory, NodeAddedTriggerIntegrationTest, TestConfigReload, TestSimPolicyCloud, DistributedQueryComponentOptimizationTest, SolrCmdDistributorTest, TestDynamicLoading, V2StandaloneTest, TestFiltering, CollectionPropsTest, TestSimComputePlanAction, CloudExitableDirectoryReaderTest, PrimUtilsTest, HttpPartitionOnCommitTest, ForceLeaderWithTlogReplicasTest, TestSchemaManager, TestMultiValuedNumericRangeQuery, TestClassicSimilarityFactory, TestBinaryResponseWriter, DefaultValueUpdateProcessorTest, TestSolrCloudWithKerberosAlt, CopyFieldTest, HdfsNNFailoverTest, TestManagedSynonymFilterFactory, HdfsTlogReplayBufferedWhileIndexingTest, AdminHandlersProxyTest, TestSolrTestCaseJ4, TestSolrConfigHandler, QueryParsingTest, TestCloudJSONFacetJoinDomain, TestCollectionAPIs, TestSolrJ, TestStressInPlaceUpdates, HttpTriggerListenerTest, SpellCheckCollatorWithCollapseTest, OverseerCollectionConfigSetProcessorTest, DistributedFacetSimpleRefinementLongTailTest, ScheduledMaintenanceTriggerTest, DistributedSuggestComponentTest, TestPerFieldSimilarityWithDefaultOverride, TestFieldTypeCollectionResource, TestExtendedDismaxParser, TestDelegationWithHadoopAuth, CloudMLTQParserTest, MetricsHistoryWithAuthIntegrationTest, ClassificationUpdateProcessorIntegrationTest, TestLocalFSCloudBackupRestore, TestSizeLimitedDistributedMap, XMLAtomicUpdateMultivalueTest, ShardSplitTest, MoveReplicaHDFSFailoverTest, TestSurroundQueryParser, CoreMergeIndexesAdminHandlerTest, BasicAuthStandaloneTest, TestSolrCachePerf, TestDistributedSearch, StressHdfsTest, TestBulkSchemaConcurrent, TestDownShardTolerantSearch, HdfsRestartWhileUpdatingTest, TestBadConfig, SolrInfoBeanTest, XmlInterpolationTest, TestDistributedStatsComponentCardinality, LeaderVoteWaitTimeoutTest, SearchHandlerTest, TestMacros, BasicDistributedZk2Test, CollectionsAPIAsyncDistributedZkTest, TestExactStatsCacheCloud, TestLeaderElectionZkExpiry, TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplica, TestPullReplicaErrorHandling, TestRandomRequestDistribution, TestRebalanceLeaders, TestTolerantUpdateProcessorRandomCloud, TestUtilizeNode, TestWaitForStateWithJettyShutdowns, TriLevelCompositeIdRoutingTest, UnloadDistributedZkTest, VMParamsZkACLAndCredentialsProvidersTest, ZkCLITest, ConcurrentDeleteAndCreateCollectionTest, TestCollectionsAPIViaSolrCloudCluster, AutoScalingHandlerTest, ExecutePlanActionTest, IndexSizeTriggerSizeEstimationTest, MetricTriggerIntegrationTest, SystemLogListenerTest, TestPolicyCloud, TriggerCooldownIntegrationTest, TriggerSetPropertiesIntegrationTest, TestSimDistributedQueue, TestSimExecutePlanAction, TestSimExtremeIndexing, TestSimTriggerIntegration, TestSimUtils, TestSnapshotCloudManager, BaseCdcrDistributedZkTest, CdcrBidirectionalTest, CdcrBootstrapTest]
> [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=CdcrBootstrapTest -Dtests.seed=AE624A5368110313 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-SD -Dtests.timezone=VST -Dtests.asserts=true -Dtests.file.encoding=UTF-8
> [junit4] ERROR 0.00s J1 | CdcrBootstrapTest (suite) <<<
> [junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
> [junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
> [junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> [junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> [junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> [junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> [junit4] > at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> [junit4] > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> [junit4] > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> [junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> [junit4] > at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> [junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> [junit4] > at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [junit4] > at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] > expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper]
> [junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
> [junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> [junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> [junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> [junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> [junit4] > at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> [junit4] > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> [junit4] > at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> [junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> [junit4] > at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> [junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> [junit4] > at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [junit4] > at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] > >
> [junit4] > at __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
> [junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
> [junit4] > at jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
> [junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] Completed [572/900 (1!)] on J1 in 157.86s, 4 tests, 1 failure, 1 skipped <<< FAILURES!
>
> [...truncated 53511 lines...]
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: builds-unsubscribe@lucene.apache.org
> For additional commands, e-mail: builds-help@lucene.apache.org
>
>
> --
> Adrien
>
>
> --
> Adrien
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 2096 - Still Unstable
Posted by Adrien Grand <jp...@gmail.com>.
I think it has little to do with the recent changes, but is more a
combination of crazy index-writer configs (low maxBufferedDocs, high
maxSegmentsPerTier) and concurrency.
On Thu, Feb 6, 2020 at 5:27 PM Adrien Grand <jp...@gmail.com> wrote:
> I'm looking into it.
>
> On Thu, Feb 6, 2020 at 7:36 AM Apache Jenkins Server <
> jenkins@builds.apache.org> wrote:
>
>> Build:
>> https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2096/
>>
>> 3 tests failed.
>> FAILED:
>> org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh
>>
>> Error Message:
>> Captured an uncaught exception in thread: Thread[id=17673,
>> name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
>>
>> Stack Trace:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17673, name=Thread-17208,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
>> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
>> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
>> at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
>> at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
>> at
>> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
>> at
>> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
>> at
>> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
>> at
>> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
>> at
>> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
>> at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
>> at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
>> at
>> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
>> at
>> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
>> at
>> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
>> at
>> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
>> at
>> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
>> at
>> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
>> at
>> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
>> at
>> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
>> at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
>> at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
>> at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
>>
>>
>> FAILED: junit.framework.TestSuite.org
>> .apache.lucene.search.TestSearcherManager
>>
>> Error Message:
>> The test or suite printed 16173 bytes to stdout and stderr, even though
>> the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it
>> completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
>>
>> Stack Trace:
>> java.lang.AssertionError: The test or suite printed 16173 bytes to stdout
>> and stderr, even though the limit was set to 8192 bytes. Increase the limit
>> with @Limit, ignore it completely with @SuppressSysoutChecks or run with
>> -Dtests.verbose=true
>> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> at
>> org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:282)
>> at
>> com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
>> at
>> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>> at
>> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>> at
>> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>> at
>> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
>> at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
>> at java.base/java.lang.Thread.run(Thread.java:834)
>>
>>
>> FAILED: junit.framework.TestSuite.org
>> .apache.solr.cloud.cdcr.CdcrBootstrapTest
>>
>> Error Message:
>> ObjectTracker found 1 object(s) that were not released!!!
>> [RawDirectoryWrapper]
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:834) expected null, but
>> was:<ObjectTracker found 1 object(s) that were not released!!!
>> [RawDirectoryWrapper]
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:834) >
>>
>> Stack Trace:
>> java.lang.AssertionError: ObjectTracker found 1 object(s) that were not
>> released!!! [RawDirectoryWrapper]
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper
>> at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>> at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:834)
>>
>> expected null, but was:<ObjectTracker found 1 object(s) that were not
>> released!!! [RawDirectoryWrapper]
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper
>> at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>> at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:834)
>>
>> >
>> at __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
>> at org.junit.Assert.fail(Assert.java:88)
>> at org.junit.Assert.failNotNull(Assert.java:755)
>> at org.junit.Assert.assertNull(Assert.java:737)
>> at
>> org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
>> at jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown
>> Source)
>> at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
>> at
>> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
>> at
>> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>> at
>> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>> at
>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>> at
>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>> at
>> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>> at
>> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>> at
>> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
>> at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
>> at java.base/java.lang.Thread.run(Thread.java:834)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 2104 lines...]
>> [junit4] Suite: org.apache.lucene.search.TestSearcherManager
>> [junit4] 2> Feb 06, 2020 3:18:49 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread:
>> Thread[Thread-17208,5,TGRP-TestSearcherManager]
>> [junit4] 2> java.lang.RuntimeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
>> [junit4] 2> at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
>> [junit4] 2> at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
>> [junit4] 2> at
>> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
>> [junit4] 2> at
>> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
>> [junit4] 2> at
>> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
>> [junit4] 2> at
>> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
>> [junit4] 2> at
>> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
>> [junit4] 2> at
>> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
>> [junit4] 2> at
>> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
>> [junit4] 2> at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
>> [junit4] 2> at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
>> [junit4] 2> at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
>> [junit4] 2>
>> [junit4] 2> Feb 06, 2020 3:18:50 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread:
>> Thread[Thread-17205,5,TGRP-TestSearcherManager]
>> [junit4] 2> org.apache.lucene.store.AlreadyClosedException: this
>> IndexWriter is closed
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
>> [junit4] 2> at
>> org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] 2> at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2>
>> [junit4] 2> Feb 06, 2020 3:18:50 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
>> Merge Thread #6,5,TGRP-TestSearcherManager]
>> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
>> Too many open files
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] 2> at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2>
>> [junit4] 2> Feb 06, 2020 3:18:50 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
>> Merge Thread #4,5,TGRP-TestSearcherManager]
>> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
>> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
>> files: this IndexWriter hit an unrecoverable exception
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] 2> Caused by:
>> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
>> files: this IndexWriter hit an unrecoverable exception
>> [junit4] 2> at
>> org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] 2> at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] 2> ... 3 more
>> [junit4] 2>
>> [junit4] 2> Feb 06, 2020 3:18:51 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
>> Merge Thread #5,5,TGRP-TestSearcherManager]
>> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] 2> at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2>
>> [junit4] 2> NOTE: download the large Jenkins line-docs file by
>> running 'ant get-jenkins-line-docs' in the lucene directory.
>> [junit4] 2> NOTE: reproduce with: ant test
>> -Dtestcase=TestSearcherManager
>> -Dtests.method=testConcurrentIndexCloseSearchAndRefresh
>> -Dtests.seed=CBAB78174F1B51A9 -Dtests.multiplier=2 -Dtests.nightly=true
>> -Dtests.slow=true
>> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
>> -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true
>> -Dtests.file.encoding=UTF-8
>> [junit4] ERROR 17.7s J0 |
>> TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh <<<
>> [junit4] > Throwable #1:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17673, name=Thread-17208,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by: java.lang.RuntimeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
>> [junit4] > at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
>> [junit4] > at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
>> [junit4] > at
>> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
>> [junit4] > at
>> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
>> [junit4] > at
>> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
>> [junit4] > at
>> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
>> [junit4] > at
>> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
>> [junit4] > at
>> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
>> [junit4] > at
>> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
>> [junit4] > at
>> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
>> [junit4] > at
>> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
>> [junit4] > at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
>> [junit4] > at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
>> [junit4] > at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)Throwable
>> #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17688, name=Lucene Merge Thread #6,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by:
>> org.apache.lucene.index.MergePolicy$MergeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
>> Too many open files
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] > at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable
>> #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17670, name=Thread-17205,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by:
>> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
>> [junit4] > at
>> org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] > at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable
>> #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17686, name=Lucene Merge Thread #4,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by:
>> org.apache.lucene.index.MergePolicy$MergeException:
>> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
>> files: this IndexWriter hit an unrecoverable exception
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] > Caused by:
>> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
>> files: this IndexWriter hit an unrecoverable exception
>> [junit4] > at
>> org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
>> [junit4] > at
>> org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
>> [junit4] > at
>> org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] > at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] > ... 3 moreThrowable #5:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17687, name=Lucene Merge Thread #5,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by:
>> org.apache.lucene.index.MergePolicy$MergeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] > at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2> NOTE: test params are: codec=Asserting(Lucene84),
>> sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@277d98a5),
>> locale=en-US, timezone=Pacific/Tarawa
>> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation
>> 11.0.4 (64-bit)/cpus=4,threads=1,free=244326912,total=536870912
>> [junit4] 2> NOTE: All tests run in this JVM:
>> [TestSameScoresWithThreads, TestLucene84PostingsFormat,
>> TestRamUsageEstimator, TestLatLonMultiLineShapeQueries,
>> TestEarlyTermination, TestSimpleFSLockFactory, TestMatchesIterator,
>> TestStressIndexing, TestMultiFields, TestIndexCommit,
>> TestDisjunctionScoreBlockBoundaryPropagator, TestMixedCodecs,
>> TestLucene50CompoundFormat, TestField, TestTotalHits, TestIndexedDISI,
>> TestSegmentCacheables, TestRegExp, TestDocIdsWriter, TestFieldReuse,
>> TestIntBlockPool, TestIndexWriterThreadsToSegments, TestReaderPool,
>> TestNoMergePolicy, TestPerFieldPostingsFormat, MultiCollectorTest,
>> TestComplexExplanationsOfNonMatches, TestLogMergePolicy, TestTermStates,
>> TestPendingDeletes, TestFeatureDoubleValues, TestMultiset,
>> TestSortRescorer, TestRollingUpdates, TestSimilarityBase,
>> TestTermVectorsWriter, TestXYRectangle2D, TestAddIndexes, TestTransactions,
>> TestSpansEnum, TestVirtualMethod, TestIndexManyDocuments, TestGeoUtils,
>> TestLatLonPointShapeQueries, TestConstantScoreQuery, TestWordlistLoader,
>> TestIntsRef, TestIndexSorting, TestMaxScoreSumPropagator, TestNeverDelete,
>> TestSegmentTermEnum, TestLucene60FieldInfoFormat, TestXYLineShapeQueries,
>> TestFieldValueQuery, TestPrefixCodedTerms, TestBM25Similarity,
>> TestAxiomaticF1LOG, TestFixedBitDocIdSet, TestUniqueTermCount,
>> TestFilterMergePolicy, TestTimeLimitingCollector, TestPhraseQuery,
>> TestMaxPosition, TestSimpleExplanationsWithFillerDocs, TestSentinelIntSet,
>> TestDocValuesRewriteMethod, TestDaciukMihovAutomatonBuilder,
>> TestNRTReaderCleanup, TestCustomNorms, TestSnapshotDeletionPolicy,
>> TestDuelingCodecs, TestStringHelper, TestIntRange,
>> TestAllFilesHaveChecksumFooter, TestCharsRef, TestPriorityQueue,
>> TestLevenshteinAutomata, TestFuzzyQuery, TestXYShape, TestBinaryTerms,
>> TestNumericUtils, TestPrefixRandom, TestAutomatonQuery,
>> TestTimSorterWorstCase, TestIndexWriterOnVMError,
>> TestXYMultiPolygonShapeQueries, TestDirectMonotonic, TestLatLonPoint,
>> TestLongValuesSource, TestNativeFSLockFactory, TestIntRangeFieldQueries,
>> TestSortedSetSortField, TestLucene50StoredFieldsFormatMergeInstance,
>> TestLucene80NormsFormatMergeInstance, Test2BPostingsBytes,
>> Test2BSortedDocValuesFixedSorted, Test2BSortedDocValuesOrds, Test2BTerms,
>> Test4GBStoredFields, TestBagOfPositions, TestBagOfPostings,
>> TestBinaryDocValuesUpdates, TestCodecHoldsOpenFiles, TestCodecs,
>> TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash,
>> TestCrashCausesCorruptIndex, TestDemoParallelLeafReader,
>> TestExitableDirectoryReader, TestFieldInfos, TestFieldInvertState,
>> TestFilterCodecReader, TestFilterDirectoryReader, TestFilterLeafReader,
>> TestForTooMuchCloning, TestForceMergeForever, TestFrozenBufferedUpdates,
>> TestIndexWriterCommit, TestIndexWriterConfig, TestIndexWriterDelete,
>> TestIndexWriterDeleteByQuery, TestIndexWriterExceptions,
>> TestIndexWriterOnDiskFull, TestOmitNorms, TestOmitPositions, TestOmitTf,
>> TestOneMergeWrappingMergePolicy, TestOrdinalMap,
>> TestParallelCompositeReader, TestParallelLeafReader,
>> TestParallelReaderEmptyIndex, TestParallelTermEnum, TestPayloads,
>> TestPayloadsOnVectors, TestPersistentSnapshotDeletionPolicy,
>> TestPointValues, TestPostingsOffsets, TestSegmentMerger, TestSegmentReader,
>> TestSegmentTermDocs, TestSortedSetDocValues, TestStressAdvance,
>> TestStressDeletes, TestSumDocFreq, TestSwappedIndexFiles, TestTermsEnum2,
>> TestThreadedForceMerge, TestTieredMergePolicy,
>> TestTragicIndexWriterDeadlock, TestTransactionRollback,
>> TestBlockMaxConjunction, TestBoolean2, TestBoolean2ScorerSupplier,
>> TestBooleanMinShouldMatch, TestBooleanOr, TestBooleanQuery,
>> TestBooleanQueryVisitSubscorers, TestBooleanRewrites, TestBooleanScorer,
>> TestBoostQuery, TestCachingCollector, TestComplexExplanations,
>> TestCustomSearcherSort, TestDateSort, TestDisjunctionMaxQuery,
>> TestLatLonPointQueries, TestLiveFieldValues, TestLongRangeFieldQueries,
>> TestSearcherManager]
>> [junit4] 2> NOTE: download the large Jenkins line-docs file by
>> running 'ant get-jenkins-line-docs' in the lucene directory.
>> [junit4] 2> NOTE: reproduce with: ant test
>> -Dtestcase=TestSearcherManager -Dtests.seed=CBAB78174F1B51A9
>> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
>> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
>> -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true
>> -Dtests.file.encoding=UTF-8
>> [junit4] ERROR 0.00s J0 | TestSearcherManager (suite) <<<
>> [junit4] > Throwable #1: java.lang.AssertionError: The test or
>> suite printed 16173 bytes to stdout and stderr, even though the limit was
>> set to 8192 bytes. Increase the limit with @Limit, ignore it completely
>> with @SuppressSysoutChecks or run with -Dtests.verbose=true
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
>> [junit4] Completed [546/552 (1!)] on J0 in 355.57s, 11 tests, 1
>> failure, 1 error <<< FAILURES!
>>
>> [...truncated 12649 lines...]
>> [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBootstrapTest
>> [junit4] 2> 5121336 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 Created dataDir:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/data-dir-186-001
>> [junit4] 2> 5121337 WARN
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11
>> [junit4] 2> 5121337 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
>> w/NUMERIC_DOCVALUES_SYSPROP=false
>> [junit4] 2> 5121337 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via:
>> @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0,
>> clientAuth=0.0/0.0)
>> [junit4] 2> 5121338 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 SecureRandom sanity checks:
>> test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
>> [junit4] 2> 5121341 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithMultipleReplicas
>> [junit4] 2> 5121341 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001
>> [junit4] 2> 5121341 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
>> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ]
>> o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
>> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ]
>> o.a.s.c.ZkTestServer Starting server
>> [junit4] 2> 5121442 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer start zk server on port:36855
>> [junit4] 2> 5121442 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:36855
>> [junit4] 2> 5121442 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36855
>> [junit4] 2> 5121442 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36855
>> [junit4] 2> 5121443 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5121452 INFO
>> (zkConnectionManagerCallback-12879-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121452 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121454 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5121455 INFO
>> (zkConnectionManagerCallback-12881-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121455 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121456 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5121457 INFO
>> (zkConnectionManagerCallback-12883-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121457 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.session DefaultSessionIdManager workerName=node0
>> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.session No SessionScavenger set, using defaults
>> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.session node0 Scavenging every 600000ms
>> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@867eda0
>> {/solr,null,AVAILABLE}
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.session DefaultSessionIdManager workerName=node0
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.session No SessionScavenger set, using defaults
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.session node0 Scavenging every 660000ms
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.session DefaultSessionIdManager workerName=node0
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.session No SessionScavenger set, using defaults
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.session node0 Scavenging every 660000ms
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d1dd263
>> {/solr,null,AVAILABLE}
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ecf5186
>> {/solr,null,AVAILABLE}
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.AbstractConnector Started ServerConnector@3473a5ca{ssl,[ssl,
>> alpn, http/1.1, h2]}{127.0.0.1:41191}
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.Server Started @5121593ms
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
>> hostPort=41191}
>> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
>> be missing or incomplete.
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter Using logger factory
>> org.apache.logging.slf4j.Log4jLoggerFactory
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.AbstractConnector Started ServerConnector@77d77ae2{ssl,[ssl,
>> alpn, http/1.1, h2]}{127.0.0.1:37757}
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.Server Started @5121593ms
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
>> hostPort=37757}
>> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
>> be missing or incomplete.
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter Using logger factory
>> org.apache.logging.slf4j.Log4jLoggerFactory
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
>> version 9.0.0
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
>> version 9.0.0
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
>> null
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
>> null
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
>> 2020-02-06T05:48:19.821958Z
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
>> 2020-02-06T05:48:19.821957Z
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.AbstractConnector Started ServerConnector@16db638b{ssl,[ssl,
>> alpn, http/1.1, h2]}{127.0.0.1:43461}
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.Server Started @5121594ms
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
>> hostPort=43461}
>> [junit4] 2> 5121567 ERROR (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
>> be missing or incomplete.
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter Using logger factory
>> org.apache.logging.slf4j.Log4jLoggerFactory
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
>> version 9.0.0
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
>> null
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
>> 2020-02-06T05:48:19.822974Z
>> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5121569 INFO
>> (zkConnectionManagerCallback-12886-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121569 INFO
>> (zkConnectionManagerCallback-12888-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121569 INFO
>> (zkConnectionManagerCallback-12890-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
>> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
>> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
>> [junit4] 2> 5122191 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
>> WhitelistHostChecker [whitelistHosts=null,
>> whitelistHostCheckingEnabled=true]
>> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@9c25133
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@9c25133[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@3dc1efed
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@3dc1efed[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122224 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
>> [junit4] 2> 5122229 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5122230 INFO
>> (zkConnectionManagerCallback-12899-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5122230 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5122332 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Waiting for
>> client to connect to ZooKeeper
>> [junit4] 2> 5122333 INFO
>> (zkConnectionManagerCallback-12901-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5122333 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Client is
>> connected to ZooKeeper
>> [junit4] 2> 5122450 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.OverseerElectionContext I am going
>> to be the leader 127.0.0.1:37757_solr
>> [junit4] 2> 5122451 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Overseer
>> (id=73197244948086791-127.0.0.1:37757_solr-n_0000000000) starting
>> [junit4] 2> 5122454 INFO
>> (OverseerStateUpdate-73197244948086791-127.0.0.1:37757_solr-n_0000000000)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Starting to work on the
>> main queue : 127.0.0.1:37757_solr
>> [junit4] 2> 5122464 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.ZkController Register node as live
>> in ZooKeeper:/live_nodes/127.0.0.1:37757_solr
>> [junit4] 2> 5122467 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.p.PackageLoader /packages.json
>> updated to version -1
>> [junit4] 2> 5122467 WARN (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.CoreContainer Not all security
>> plugins configured! authentication=disabled authorization=disabled. Solr
>> is only as secure as you make it. Consider configuring
>> authentication/authorization before exposing Solr to users internal or
>> external. See https://s.apache.org/solrsecurity for more info
>> [junit4] 2> 5122475 INFO (zkCallback-12900-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
>> [junit4] 2> 5122500 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
>> collection, keeping metrics history in memory.
>> [junit4] 2> 5122523 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_37757.solr.node' (registry 'solr.node') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_37757.solr.jvm' (registry 'solr.jvm') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_37757.solr.jetty' (registry 'solr.jetty') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122533 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
>> definitions underneath
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node1/.
>> [junit4] 2> 5122737 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
>> WhitelistHostChecker [whitelistHosts=null,
>> whitelistHostCheckingEnabled=true]
>> [junit4] 2> 5122738 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@2d6e218d
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122739 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@2d6e218d[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@1ed7f896
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@1ed7f896[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122743 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
>> [junit4] 2> 5122744 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5122746 INFO
>> (zkConnectionManagerCallback-12911-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5122746 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5122848 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Waiting for
>> client to connect to ZooKeeper
>> [junit4] 2> 5122852 INFO
>> (zkConnectionManagerCallback-12913-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5122852 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Client is
>> connected to ZooKeeper
>> [junit4] 2> 5122864 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ZkStateReader Updated live nodes
>> from ZooKeeper... (0) -> (1)
>> [junit4] 2> 5122868 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Publish
>> node=127.0.0.1:43461_solr as DOWN
>> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.TransientSolrCoreCacheDefault
>> Allocating transient cache for 2147483647 transient cores
>> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Register node as live
>> in ZooKeeper:/live_nodes/127.0.0.1:43461_solr
>> [junit4] 2> 5122870 INFO (zkCallback-12900-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
>> [junit4] 2> 5122870 INFO (zkCallback-12912-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
>> [junit4] 2> 5122871 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.p.PackageLoader /packages.json
>> updated to version -1
>> [junit4] 2> 5122871 WARN (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.CoreContainer Not all security
>> plugins configured! authentication=disabled authorization=disabled. Solr
>> is only as secure as you make it. Consider configuring
>> authentication/authorization before exposing Solr to users internal or
>> external. See https://s.apache.org/solrsecurity for more info
>> [junit4] 2> 5122894 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
>> collection, keeping metrics history in memory.
>> [junit4] 2> 5122918 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_43461.solr.node' (registry 'solr.node') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122929 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_43461.solr.jvm' (registry 'solr.jvm') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122930 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_43461.solr.jetty' (registry 'solr.jetty') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122931 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
>> definitions underneath
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node3/.
>> [junit4] 2> 5123682 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
>> WhitelistHostChecker [whitelistHosts=null,
>> whitelistHostCheckingEnabled=true]
>> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@32f87da4
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@32f87da4[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@32434175
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@32434175[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5123687 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
>> [junit4] 2> 5123688 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5123693 INFO
>> (zkConnectionManagerCallback-12923-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5123693 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5123802 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Waiting for
>> client to connect to ZooKeeper
>> [junit4] 2> 5123803 INFO
>> (zkConnectionManagerCallback-12925-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5123803 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Client is
>> connected to ZooKeeper
>> [junit4] 2> 5123810 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ZkStateReader Updated live nodes
>> from ZooKeeper... (0) -> (2)
>> [junit4] 2> 5123813 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Publish
>> node=127.0.0.1:41191_solr as DOWN
>> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.TransientSolrCoreCacheDefault
>> Allocating transient cache for 2147483647 transient cores
>> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Register node as live
>> in ZooKeeper:/live_nodes/127.0.0.1:41191_solr
>> [junit4] 2> 5123815 INFO (zkCallback-12912-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>> [junit4] 2> 5123815 INFO (zkCallback-12900-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>> [junit4] 2> 5123817 INFO (zkCallback-12924-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>> [junit4] 2> 5123817 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.p.PackageLoader /packages.json
>> updated to version -1
>> [junit4] 2> 5123818 WARN (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.CoreContainer Not all security
>> plugins configured! authentication=disabled authorization=disabled. Solr
>> is only as secure as you make it. Consider configuring
>> authentication/authorization before exposing Solr to users internal or
>> external. See https://s.apache.org/solrsecurity for more info
>> [junit4] 2> 5123839 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
>> collection, keeping metrics history in memory.
>> [junit4] 2> 5123860 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_41191.solr.node' (registry 'solr.node') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_41191.solr.jvm' (registry 'solr.jvm') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_41191.solr.jetty' (registry 'solr.jetty') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5123869 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
>> definitions underneath
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node2/.
>> [junit4] 2> 5123900 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=3
>> [junit4] 2> 5123901 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5123902 INFO
>> (zkConnectionManagerCallback-12934-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5123903 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5123904 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) ->
>> (3)
>> [junit4] 2> 5123905 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at
>> 127.0.0.1:36855/solr ready
>> [junit4] 1> Target zkHost = 127.0.0.1:36855/solr
>> [junit4] 2> 5123905 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-source-001
>> [junit4] 2> 5123906 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
>> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ]
>> o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
>> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ]
>> o.a.s.c.ZkTestServer Starting server
>> [junit4] 2> 5124006 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer start zk server on port:40811
>> [junit4] 2> 5124006 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:40811
>> [junit4] 2> 5124006 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40811
>> [junit4] 2> 5124006 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40811
>> [junit4] 2> 5124009 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5124011 INFO
>> (zkConnectionManagerCallback-12936-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5124011 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5124013 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5124014 INFO
>> (zkConnectionManagerCallback-12938-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5124014 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5124015 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5124015 INFO
>> (zkConnectionManagerCallback-12940-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5124015 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-3) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-2) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-1) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>>
>>
>> [...truncated too long message...]
>>
>> p(Native Method)
>> [junit4] 2>
>> app//com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
>> [junit4] 2>
>> [junit4] 2> 5279049 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4
>> ------------------------------------------------------- Done waiting for
>> tracked resources to be released
>> [junit4] 2> NOTE: test params are: codec=SimpleText,
>> sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@8a39653),
>> locale=en-SD, timezone=VST
>> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation
>> 11.0.4 (64-bit)/cpus=4,threads=1,free=386818192,total=536870912
>> [junit4] 2> NOTE: All tests run in this JVM: [AuditLoggerPluginTest,
>> CdcrUpdateLogTest, SolrCloudExampleTest, ZookeeperStatusHandlerTest,
>> UUIDFieldTest, NodeLostTriggerTest,
>> TestOverriddenPrefixQueryForCustomFieldType, TestRandomFaceting,
>> TestIBSimilarityFactory, TestUtils, UpdateLogTest, TestConfigOverlay,
>> PeerSyncWithBufferUpdatesTest, TestHashQParserPlugin,
>> TestLegacyBM25SimilarityFactory, CachingDirectoryFactoryTest,
>> SortByFunctionTest, TestZkChroot, IgnoreLargeDocumentProcessorFactoryTest,
>> TestPHPSerializedResponseWriter, TestInPlaceUpdatesRequiredField,
>> TestJettySolrRunner, TestSerializedLuceneMatchVersion, DOMUtilTest,
>> CoreAdminHandlerTest, TestManagedResource, TestSolrIndexConfig,
>> BJQParserTest, TestSimLargeCluster, TestStressThreadBackup,
>> AsyncCallRequestStatusResponseTest, TestImplicitCoreProperties,
>> DistributedFacetPivotLongTailTest, WrapperMergePolicyFactoryTest,
>> TestConfigSetsAPIExclusivity, TestRTimerTree, SolrJmxReporterCloudTest,
>> QueryElevationComponentTest, NotRequiredUniqueKeyTest,
>> ChaosMonkeyNothingIsSafeTest, MBeansHandlerTest,
>> TestHighlightDedupGrouping, ZkShardTermsTest, TestJmxIntegration,
>> CurrencyRangeFacetCloudTest, ShowFileRequestHandlerTest, TestRTGBase,
>> TestCloudSearcherWarming, TestRawTransformer,
>> SignificantTermsQParserPluginTest, MinimalSchemaTest,
>> TestGeoJSONResponseWriter, AutoAddReplicasIntegrationTest,
>> TestChildDocTransformer, TestStandardQParsers, TestNestedDocsSort,
>> TestReRankQParserPlugin, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest,
>> SolrCoreCheckLockOnStartupTest, TestSSLRandomization,
>> TestLegacyNumericRangeQueryBuilder, HdfsBasicDistributedZkTest,
>> OverseerSolrResponseTest, TestPKIAuthenticationPlugin,
>> DistributedFacetPivotWhiteBoxTest, TermVectorComponentDistributedTest,
>> Tagger2Test, PeerSyncWithLeaderTest, TestLRUStatsCacheCloud,
>> ZkStateReaderTest, LeaderFailoverAfterPartitionTest,
>> MultiSolrCloudTestCaseTest, LeaderElectionContextKeyTest, TestTlogReplica,
>> SearchRateTriggerIntegrationTest, TestCollationField,
>> TestHighFrequencyDictionaryFactory, NodeAddedTriggerIntegrationTest,
>> TestConfigReload, TestSimPolicyCloud,
>> DistributedQueryComponentOptimizationTest, SolrCmdDistributorTest,
>> TestDynamicLoading, V2StandaloneTest, TestFiltering, CollectionPropsTest,
>> TestSimComputePlanAction, CloudExitableDirectoryReaderTest, PrimUtilsTest,
>> HttpPartitionOnCommitTest, ForceLeaderWithTlogReplicasTest,
>> TestSchemaManager, TestMultiValuedNumericRangeQuery,
>> TestClassicSimilarityFactory, TestBinaryResponseWriter,
>> DefaultValueUpdateProcessorTest, TestSolrCloudWithKerberosAlt,
>> CopyFieldTest, HdfsNNFailoverTest, TestManagedSynonymFilterFactory,
>> HdfsTlogReplayBufferedWhileIndexingTest, AdminHandlersProxyTest,
>> TestSolrTestCaseJ4, TestSolrConfigHandler, QueryParsingTest,
>> TestCloudJSONFacetJoinDomain, TestCollectionAPIs, TestSolrJ,
>> TestStressInPlaceUpdates, HttpTriggerListenerTest,
>> SpellCheckCollatorWithCollapseTest,
>> OverseerCollectionConfigSetProcessorTest,
>> DistributedFacetSimpleRefinementLongTailTest,
>> ScheduledMaintenanceTriggerTest, DistributedSuggestComponentTest,
>> TestPerFieldSimilarityWithDefaultOverride, TestFieldTypeCollectionResource,
>> TestExtendedDismaxParser, TestDelegationWithHadoopAuth,
>> CloudMLTQParserTest, MetricsHistoryWithAuthIntegrationTest,
>> ClassificationUpdateProcessorIntegrationTest,
>> TestLocalFSCloudBackupRestore, TestSizeLimitedDistributedMap,
>> XMLAtomicUpdateMultivalueTest, ShardSplitTest, MoveReplicaHDFSFailoverTest,
>> TestSurroundQueryParser, CoreMergeIndexesAdminHandlerTest,
>> BasicAuthStandaloneTest, TestSolrCachePerf, TestDistributedSearch,
>> StressHdfsTest, TestBulkSchemaConcurrent, TestDownShardTolerantSearch,
>> HdfsRestartWhileUpdatingTest, TestBadConfig, SolrInfoBeanTest,
>> XmlInterpolationTest, TestDistributedStatsComponentCardinality,
>> LeaderVoteWaitTimeoutTest, SearchHandlerTest, TestMacros,
>> BasicDistributedZk2Test, CollectionsAPIAsyncDistributedZkTest,
>> TestExactStatsCacheCloud, TestLeaderElectionZkExpiry,
>> TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplica,
>> TestPullReplicaErrorHandling, TestRandomRequestDistribution,
>> TestRebalanceLeaders, TestTolerantUpdateProcessorRandomCloud,
>> TestUtilizeNode, TestWaitForStateWithJettyShutdowns,
>> TriLevelCompositeIdRoutingTest, UnloadDistributedZkTest,
>> VMParamsZkACLAndCredentialsProvidersTest, ZkCLITest,
>> ConcurrentDeleteAndCreateCollectionTest,
>> TestCollectionsAPIViaSolrCloudCluster, AutoScalingHandlerTest,
>> ExecutePlanActionTest, IndexSizeTriggerSizeEstimationTest,
>> MetricTriggerIntegrationTest, SystemLogListenerTest, TestPolicyCloud,
>> TriggerCooldownIntegrationTest, TriggerSetPropertiesIntegrationTest,
>> TestSimDistributedQueue, TestSimExecutePlanAction, TestSimExtremeIndexing,
>> TestSimTriggerIntegration, TestSimUtils, TestSnapshotCloudManager,
>> BaseCdcrDistributedZkTest, CdcrBidirectionalTest, CdcrBootstrapTest]
>> [junit4] 2> NOTE: download the large Jenkins line-docs file by
>> running 'ant get-jenkins-line-docs' in the lucene directory.
>> [junit4] 2> NOTE: reproduce with: ant test
>> -Dtestcase=CdcrBootstrapTest -Dtests.seed=AE624A5368110313
>> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
>> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
>> -Dtests.locale=en-SD -Dtests.timezone=VST -Dtests.asserts=true
>> -Dtests.file.encoding=UTF-8
>> [junit4] ERROR 0.00s J1 | CdcrBootstrapTest (suite) <<<
>> [junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker
>> found 1 object(s) that were not released!!! [RawDirectoryWrapper]
>> [junit4] >
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper
>> [junit4] > at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> [junit4] > at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> [junit4] > at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> [junit4] > at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> [junit4] > at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> [junit4] > at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> [junit4] > at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> [junit4] > at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> [junit4] > at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>> [junit4] > at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> [junit4] > at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> [junit4] > at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
>> [junit4] > expected null, but was:<ObjectTracker found 1 object(s)
>> that were not released!!! [RawDirectoryWrapper]
>> [junit4] >
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper
>> [junit4] > at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> [junit4] > at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> [junit4] > at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> [junit4] > at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> [junit4] > at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> [junit4] > at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> [junit4] > at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> [junit4] > at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> [junit4] > at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>> [junit4] > at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> [junit4] > at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> [junit4] > at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
>> [junit4] > >
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
>> [junit4] > at
>> org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
>> [junit4] > at
>> jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
>> [junit4] > at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> [junit4] > at
>> java.base/java.lang.reflect.Method.invoke(Method.java:566)
>> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
>> [junit4] Completed [572/900 (1!)] on J1 in 157.86s, 4 tests, 1
>> failure, 1 skipped <<< FAILURES!
>>
>> [...truncated 53511 lines...]
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: builds-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: builds-help@lucene.apache.org
>
>
>
> --
> Adrien
>
--
Adrien
Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 2096 - Still Unstable
Posted by Adrien Grand <jp...@gmail.com>.
I think it has little to do with the recent changes, but is more a
combination of crazy index-writer configs (low maxBufferedDocs, high
maxSegmentsPerTier) and concurrency.
On Thu, Feb 6, 2020 at 5:27 PM Adrien Grand <jp...@gmail.com> wrote:
> I'm looking into it.
>
> On Thu, Feb 6, 2020 at 7:36 AM Apache Jenkins Server <
> jenkins@builds.apache.org> wrote:
>
>> Build:
>> https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2096/
>>
>> 3 tests failed.
>> FAILED:
>> org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh
>>
>> Error Message:
>> Captured an uncaught exception in thread: Thread[id=17673,
>> name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
>>
>> Stack Trace:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17673, name=Thread-17208,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
>> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
>> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
>> at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
>> at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
>> at
>> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
>> at
>> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
>> at
>> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
>> at
>> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
>> at
>> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
>> at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
>> at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
>> at
>> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
>> at
>> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
>> at
>> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
>> at
>> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
>> at
>> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
>> at
>> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
>> at
>> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
>> at
>> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
>> at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
>> at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
>> at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
>>
>>
>> FAILED: junit.framework.TestSuite.org
>> .apache.lucene.search.TestSearcherManager
>>
>> Error Message:
>> The test or suite printed 16173 bytes to stdout and stderr, even though
>> the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it
>> completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
>>
>> Stack Trace:
>> java.lang.AssertionError: The test or suite printed 16173 bytes to stdout
>> and stderr, even though the limit was set to 8192 bytes. Increase the limit
>> with @Limit, ignore it completely with @SuppressSysoutChecks or run with
>> -Dtests.verbose=true
>> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> at
>> org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:282)
>> at
>> com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
>> at
>> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>> at
>> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>> at
>> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>> at
>> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
>> at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
>> at java.base/java.lang.Thread.run(Thread.java:834)
>>
>>
>> FAILED: junit.framework.TestSuite.org
>> .apache.solr.cloud.cdcr.CdcrBootstrapTest
>>
>> Error Message:
>> ObjectTracker found 1 object(s) that were not released!!!
>> [RawDirectoryWrapper]
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:834) expected null, but
>> was:<ObjectTracker found 1 object(s) that were not released!!!
>> [RawDirectoryWrapper]
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:834) >
>>
>> Stack Trace:
>> java.lang.AssertionError: ObjectTracker found 1 object(s) that were not
>> released!!! [RawDirectoryWrapper]
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper
>> at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>> at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:834)
>>
>> expected null, but was:<ObjectTracker found 1 object(s) that were not
>> released!!! [RawDirectoryWrapper]
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper
>> at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>> at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:834)
>>
>> >
>> at __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
>> at org.junit.Assert.fail(Assert.java:88)
>> at org.junit.Assert.failNotNull(Assert.java:755)
>> at org.junit.Assert.assertNull(Assert.java:737)
>> at
>> org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
>> at jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown
>> Source)
>> at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
>> at
>> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
>> at
>> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>> at
>> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>> at
>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>> at
>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>> at
>> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>> at
>> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>> at
>> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>> at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
>> at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
>> at java.base/java.lang.Thread.run(Thread.java:834)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 2104 lines...]
>> [junit4] Suite: org.apache.lucene.search.TestSearcherManager
>> [junit4] 2> Feb 06, 2020 3:18:49 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread:
>> Thread[Thread-17208,5,TGRP-TestSearcherManager]
>> [junit4] 2> java.lang.RuntimeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
>> [junit4] 2> at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
>> [junit4] 2> at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
>> [junit4] 2> at
>> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
>> [junit4] 2> at
>> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
>> [junit4] 2> at
>> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
>> [junit4] 2> at
>> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
>> [junit4] 2> at
>> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
>> [junit4] 2> at
>> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
>> [junit4] 2> at
>> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
>> [junit4] 2> at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
>> [junit4] 2> at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
>> [junit4] 2> at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
>> [junit4] 2>
>> [junit4] 2> Feb 06, 2020 3:18:50 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread:
>> Thread[Thread-17205,5,TGRP-TestSearcherManager]
>> [junit4] 2> org.apache.lucene.store.AlreadyClosedException: this
>> IndexWriter is closed
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
>> [junit4] 2> at
>> org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] 2> at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2>
>> [junit4] 2> Feb 06, 2020 3:18:50 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
>> Merge Thread #6,5,TGRP-TestSearcherManager]
>> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
>> Too many open files
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] 2> at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2>
>> [junit4] 2> Feb 06, 2020 3:18:50 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
>> Merge Thread #4,5,TGRP-TestSearcherManager]
>> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
>> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
>> files: this IndexWriter hit an unrecoverable exception
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] 2> Caused by:
>> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
>> files: this IndexWriter hit an unrecoverable exception
>> [junit4] 2> at
>> org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] 2> at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] 2> ... 3 more
>> [junit4] 2>
>> [junit4] 2> Feb 06, 2020 3:18:51 PM
>> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
>> uncaughtException
>> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
>> Merge Thread #5,5,TGRP-TestSearcherManager]
>> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] 2> at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] 2> Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] 2> at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] 2> at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] 2> at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] 2> at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] 2> at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] 2> at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] 2> at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] 2> at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] 2> at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] 2> at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] 2> at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2>
>> [junit4] 2> NOTE: download the large Jenkins line-docs file by
>> running 'ant get-jenkins-line-docs' in the lucene directory.
>> [junit4] 2> NOTE: reproduce with: ant test
>> -Dtestcase=TestSearcherManager
>> -Dtests.method=testConcurrentIndexCloseSearchAndRefresh
>> -Dtests.seed=CBAB78174F1B51A9 -Dtests.multiplier=2 -Dtests.nightly=true
>> -Dtests.slow=true
>> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
>> -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true
>> -Dtests.file.encoding=UTF-8
>> [junit4] ERROR 17.7s J0 |
>> TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh <<<
>> [junit4] > Throwable #1:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17673, name=Thread-17208,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by: java.lang.RuntimeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
>> [junit4] > at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
>> [junit4] > at
>> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
>> [junit4] > at
>> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
>> [junit4] > at
>> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
>> [junit4] > at
>> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
>> [junit4] > at
>> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
>> [junit4] > at
>> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
>> [junit4] > at
>> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
>> [junit4] > at
>> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
>> [junit4] > at
>> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
>> [junit4] > at
>> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
>> [junit4] > at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
>> [junit4] > at
>> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
>> [junit4] > at
>> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)Throwable
>> #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17688, name=Lucene Merge Thread #6,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by:
>> org.apache.lucene.index.MergePolicy$MergeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
>> Too many open files
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] > at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable
>> #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17670, name=Thread-17205,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by:
>> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
>> [junit4] > at
>> org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] > at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable
>> #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17686, name=Lucene Merge Thread #4,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by:
>> org.apache.lucene.index.MergePolicy$MergeException:
>> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
>> files: this IndexWriter hit an unrecoverable exception
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] > Caused by:
>> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
>> files: this IndexWriter hit an unrecoverable exception
>> [junit4] > at
>> org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
>> [junit4] > at
>> org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
>> [junit4] > at
>> org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] > at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] > ... 3 moreThrowable #5:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=17687, name=Lucene Merge Thread #5,
>> state=RUNNABLE, group=TGRP-TestSearcherManager]
>> [junit4] > Caused by:
>> org.apache.lucene.index.MergePolicy$MergeException:
>> java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
>> [junit4] > Caused by: java.nio.file.FileSystemException:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
>> Too many open files
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>> [junit4] > at
>> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
>> [junit4] > at
>> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
>> [junit4] > at
>> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
>> [junit4] > at
>> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
>> [junit4] > at
>> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
>> [junit4] > at
>> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
>> [junit4] > at
>> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
>> [junit4] > at
>> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
>> [junit4] > at
>> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
>> [junit4] > at
>> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
>> [junit4] > at
>> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
>> [junit4] > at
>> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
>> [junit4] 2> NOTE: test params are: codec=Asserting(Lucene84),
>> sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@277d98a5),
>> locale=en-US, timezone=Pacific/Tarawa
>> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation
>> 11.0.4 (64-bit)/cpus=4,threads=1,free=244326912,total=536870912
>> [junit4] 2> NOTE: All tests run in this JVM:
>> [TestSameScoresWithThreads, TestLucene84PostingsFormat,
>> TestRamUsageEstimator, TestLatLonMultiLineShapeQueries,
>> TestEarlyTermination, TestSimpleFSLockFactory, TestMatchesIterator,
>> TestStressIndexing, TestMultiFields, TestIndexCommit,
>> TestDisjunctionScoreBlockBoundaryPropagator, TestMixedCodecs,
>> TestLucene50CompoundFormat, TestField, TestTotalHits, TestIndexedDISI,
>> TestSegmentCacheables, TestRegExp, TestDocIdsWriter, TestFieldReuse,
>> TestIntBlockPool, TestIndexWriterThreadsToSegments, TestReaderPool,
>> TestNoMergePolicy, TestPerFieldPostingsFormat, MultiCollectorTest,
>> TestComplexExplanationsOfNonMatches, TestLogMergePolicy, TestTermStates,
>> TestPendingDeletes, TestFeatureDoubleValues, TestMultiset,
>> TestSortRescorer, TestRollingUpdates, TestSimilarityBase,
>> TestTermVectorsWriter, TestXYRectangle2D, TestAddIndexes, TestTransactions,
>> TestSpansEnum, TestVirtualMethod, TestIndexManyDocuments, TestGeoUtils,
>> TestLatLonPointShapeQueries, TestConstantScoreQuery, TestWordlistLoader,
>> TestIntsRef, TestIndexSorting, TestMaxScoreSumPropagator, TestNeverDelete,
>> TestSegmentTermEnum, TestLucene60FieldInfoFormat, TestXYLineShapeQueries,
>> TestFieldValueQuery, TestPrefixCodedTerms, TestBM25Similarity,
>> TestAxiomaticF1LOG, TestFixedBitDocIdSet, TestUniqueTermCount,
>> TestFilterMergePolicy, TestTimeLimitingCollector, TestPhraseQuery,
>> TestMaxPosition, TestSimpleExplanationsWithFillerDocs, TestSentinelIntSet,
>> TestDocValuesRewriteMethod, TestDaciukMihovAutomatonBuilder,
>> TestNRTReaderCleanup, TestCustomNorms, TestSnapshotDeletionPolicy,
>> TestDuelingCodecs, TestStringHelper, TestIntRange,
>> TestAllFilesHaveChecksumFooter, TestCharsRef, TestPriorityQueue,
>> TestLevenshteinAutomata, TestFuzzyQuery, TestXYShape, TestBinaryTerms,
>> TestNumericUtils, TestPrefixRandom, TestAutomatonQuery,
>> TestTimSorterWorstCase, TestIndexWriterOnVMError,
>> TestXYMultiPolygonShapeQueries, TestDirectMonotonic, TestLatLonPoint,
>> TestLongValuesSource, TestNativeFSLockFactory, TestIntRangeFieldQueries,
>> TestSortedSetSortField, TestLucene50StoredFieldsFormatMergeInstance,
>> TestLucene80NormsFormatMergeInstance, Test2BPostingsBytes,
>> Test2BSortedDocValuesFixedSorted, Test2BSortedDocValuesOrds, Test2BTerms,
>> Test4GBStoredFields, TestBagOfPositions, TestBagOfPostings,
>> TestBinaryDocValuesUpdates, TestCodecHoldsOpenFiles, TestCodecs,
>> TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash,
>> TestCrashCausesCorruptIndex, TestDemoParallelLeafReader,
>> TestExitableDirectoryReader, TestFieldInfos, TestFieldInvertState,
>> TestFilterCodecReader, TestFilterDirectoryReader, TestFilterLeafReader,
>> TestForTooMuchCloning, TestForceMergeForever, TestFrozenBufferedUpdates,
>> TestIndexWriterCommit, TestIndexWriterConfig, TestIndexWriterDelete,
>> TestIndexWriterDeleteByQuery, TestIndexWriterExceptions,
>> TestIndexWriterOnDiskFull, TestOmitNorms, TestOmitPositions, TestOmitTf,
>> TestOneMergeWrappingMergePolicy, TestOrdinalMap,
>> TestParallelCompositeReader, TestParallelLeafReader,
>> TestParallelReaderEmptyIndex, TestParallelTermEnum, TestPayloads,
>> TestPayloadsOnVectors, TestPersistentSnapshotDeletionPolicy,
>> TestPointValues, TestPostingsOffsets, TestSegmentMerger, TestSegmentReader,
>> TestSegmentTermDocs, TestSortedSetDocValues, TestStressAdvance,
>> TestStressDeletes, TestSumDocFreq, TestSwappedIndexFiles, TestTermsEnum2,
>> TestThreadedForceMerge, TestTieredMergePolicy,
>> TestTragicIndexWriterDeadlock, TestTransactionRollback,
>> TestBlockMaxConjunction, TestBoolean2, TestBoolean2ScorerSupplier,
>> TestBooleanMinShouldMatch, TestBooleanOr, TestBooleanQuery,
>> TestBooleanQueryVisitSubscorers, TestBooleanRewrites, TestBooleanScorer,
>> TestBoostQuery, TestCachingCollector, TestComplexExplanations,
>> TestCustomSearcherSort, TestDateSort, TestDisjunctionMaxQuery,
>> TestLatLonPointQueries, TestLiveFieldValues, TestLongRangeFieldQueries,
>> TestSearcherManager]
>> [junit4] 2> NOTE: download the large Jenkins line-docs file by
>> running 'ant get-jenkins-line-docs' in the lucene directory.
>> [junit4] 2> NOTE: reproduce with: ant test
>> -Dtestcase=TestSearcherManager -Dtests.seed=CBAB78174F1B51A9
>> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
>> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
>> -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true
>> -Dtests.file.encoding=UTF-8
>> [junit4] ERROR 0.00s J0 | TestSearcherManager (suite) <<<
>> [junit4] > Throwable #1: java.lang.AssertionError: The test or
>> suite printed 16173 bytes to stdout and stderr, even though the limit was
>> set to 8192 bytes. Increase the limit with @Limit, ignore it completely
>> with @SuppressSysoutChecks or run with -Dtests.verbose=true
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
>> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
>> [junit4] Completed [546/552 (1!)] on J0 in 355.57s, 11 tests, 1
>> failure, 1 error <<< FAILURES!
>>
>> [...truncated 12649 lines...]
>> [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBootstrapTest
>> [junit4] 2> 5121336 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 Created dataDir:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/data-dir-186-001
>> [junit4] 2> 5121337 WARN
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11
>> [junit4] 2> 5121337 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
>> w/NUMERIC_DOCVALUES_SYSPROP=false
>> [junit4] 2> 5121337 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via:
>> @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0,
>> clientAuth=0.0/0.0)
>> [junit4] 2> 5121338 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4 SecureRandom sanity checks:
>> test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
>> [junit4] 2> 5121341 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithMultipleReplicas
>> [junit4] 2> 5121341 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001
>> [junit4] 2> 5121341 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
>> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ]
>> o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
>> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ]
>> o.a.s.c.ZkTestServer Starting server
>> [junit4] 2> 5121442 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer start zk server on port:36855
>> [junit4] 2> 5121442 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:36855
>> [junit4] 2> 5121442 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36855
>> [junit4] 2> 5121442 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36855
>> [junit4] 2> 5121443 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5121452 INFO
>> (zkConnectionManagerCallback-12879-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121452 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121454 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5121455 INFO
>> (zkConnectionManagerCallback-12881-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121455 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121456 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5121457 INFO
>> (zkConnectionManagerCallback-12883-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121457 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.session DefaultSessionIdManager workerName=node0
>> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.session No SessionScavenger set, using defaults
>> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.session node0 Scavenging every 600000ms
>> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@867eda0
>> {/solr,null,AVAILABLE}
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.session DefaultSessionIdManager workerName=node0
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.session No SessionScavenger set, using defaults
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.session node0 Scavenging every 660000ms
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.session DefaultSessionIdManager workerName=node0
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.session No SessionScavenger set, using defaults
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.session node0 Scavenging every 660000ms
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d1dd263
>> {/solr,null,AVAILABLE}
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ecf5186
>> {/solr,null,AVAILABLE}
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.AbstractConnector Started ServerConnector@3473a5ca{ssl,[ssl,
>> alpn, http/1.1, h2]}{127.0.0.1:41191}
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.s.Server Started @5121593ms
>> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
>> hostPort=41191}
>> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
>> be missing or incomplete.
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter Using logger factory
>> org.apache.logging.slf4j.Log4jLoggerFactory
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.AbstractConnector Started ServerConnector@77d77ae2{ssl,[ssl,
>> alpn, http/1.1, h2]}{127.0.0.1:37757}
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.s.Server Started @5121593ms
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
>> hostPort=37757}
>> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
>> be missing or incomplete.
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter Using logger factory
>> org.apache.logging.slf4j.Log4jLoggerFactory
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
>> version 9.0.0
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
>> version 9.0.0
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
>> null
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
>> null
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
>> 2020-02-06T05:48:19.821958Z
>> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
>> 2020-02-06T05:48:19.821957Z
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.AbstractConnector Started ServerConnector@16db638b{ssl,[ssl,
>> alpn, http/1.1, h2]}{127.0.0.1:43461}
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.s.Server Started @5121594ms
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
>> hostPort=43461}
>> [junit4] 2> 5121567 ERROR (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
>> be missing or incomplete.
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter Using logger factory
>> org.apache.logging.slf4j.Log4jLoggerFactory
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
>> version 9.0.0
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
>> null
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
>> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
>> 2020-02-06T05:48:19.822974Z
>> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5121569 INFO
>> (zkConnectionManagerCallback-12886-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121569 INFO
>> (zkConnectionManagerCallback-12888-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121569 INFO
>> (zkConnectionManagerCallback-12890-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
>> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
>> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
>> [junit4] 2> 5122191 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
>> WhitelistHostChecker [whitelistHosts=null,
>> whitelistHostCheckingEnabled=true]
>> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@9c25133
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@9c25133[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@3dc1efed
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@3dc1efed[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122224 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
>> [junit4] 2> 5122229 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5122230 INFO
>> (zkConnectionManagerCallback-12899-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5122230 INFO (jetty-launcher-12884-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5122332 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Waiting for
>> client to connect to ZooKeeper
>> [junit4] 2> 5122333 INFO
>> (zkConnectionManagerCallback-12901-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5122333 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Client is
>> connected to ZooKeeper
>> [junit4] 2> 5122450 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.OverseerElectionContext I am going
>> to be the leader 127.0.0.1:37757_solr
>> [junit4] 2> 5122451 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Overseer
>> (id=73197244948086791-127.0.0.1:37757_solr-n_0000000000) starting
>> [junit4] 2> 5122454 INFO
>> (OverseerStateUpdate-73197244948086791-127.0.0.1:37757_solr-n_0000000000)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Starting to work on the
>> main queue : 127.0.0.1:37757_solr
>> [junit4] 2> 5122464 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.ZkController Register node as live
>> in ZooKeeper:/live_nodes/127.0.0.1:37757_solr
>> [junit4] 2> 5122467 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.p.PackageLoader /packages.json
>> updated to version -1
>> [junit4] 2> 5122467 WARN (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.CoreContainer Not all security
>> plugins configured! authentication=disabled authorization=disabled. Solr
>> is only as secure as you make it. Consider configuring
>> authentication/authorization before exposing Solr to users internal or
>> external. See https://s.apache.org/solrsecurity for more info
>> [junit4] 2> 5122475 INFO (zkCallback-12900-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
>> [junit4] 2> 5122500 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
>> collection, keeping metrics history in memory.
>> [junit4] 2> 5122523 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_37757.solr.node' (registry 'solr.node') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_37757.solr.jvm' (registry 'solr.jvm') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_37757.solr.jetty' (registry 'solr.jetty') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122533 INFO (jetty-launcher-12884-thread-1)
>> [n:127.0.0.1:37757_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
>> definitions underneath
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node1/.
>> [junit4] 2> 5122737 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
>> WhitelistHostChecker [whitelistHosts=null,
>> whitelistHostCheckingEnabled=true]
>> [junit4] 2> 5122738 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@2d6e218d
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122739 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@2d6e218d[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@1ed7f896
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@1ed7f896[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5122743 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
>> [junit4] 2> 5122744 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5122746 INFO
>> (zkConnectionManagerCallback-12911-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5122746 INFO (jetty-launcher-12884-thread-3) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5122848 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Waiting for
>> client to connect to ZooKeeper
>> [junit4] 2> 5122852 INFO
>> (zkConnectionManagerCallback-12913-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5122852 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Client is
>> connected to ZooKeeper
>> [junit4] 2> 5122864 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ZkStateReader Updated live nodes
>> from ZooKeeper... (0) -> (1)
>> [junit4] 2> 5122868 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Publish
>> node=127.0.0.1:43461_solr as DOWN
>> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.TransientSolrCoreCacheDefault
>> Allocating transient cache for 2147483647 transient cores
>> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Register node as live
>> in ZooKeeper:/live_nodes/127.0.0.1:43461_solr
>> [junit4] 2> 5122870 INFO (zkCallback-12900-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
>> [junit4] 2> 5122870 INFO (zkCallback-12912-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
>> [junit4] 2> 5122871 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.p.PackageLoader /packages.json
>> updated to version -1
>> [junit4] 2> 5122871 WARN (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.CoreContainer Not all security
>> plugins configured! authentication=disabled authorization=disabled. Solr
>> is only as secure as you make it. Consider configuring
>> authentication/authorization before exposing Solr to users internal or
>> external. See https://s.apache.org/solrsecurity for more info
>> [junit4] 2> 5122894 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
>> collection, keeping metrics history in memory.
>> [junit4] 2> 5122918 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_43461.solr.node' (registry 'solr.node') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122929 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_43461.solr.jvm' (registry 'solr.jvm') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122930 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_43461.solr.jetty' (registry 'solr.jetty') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5122931 INFO (jetty-launcher-12884-thread-3)
>> [n:127.0.0.1:43461_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
>> definitions underneath
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node3/.
>> [junit4] 2> 5123682 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
>> WhitelistHostChecker [whitelistHosts=null,
>> whitelistHostCheckingEnabled=true]
>> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@32f87da4
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@32f87da4[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.u.s.S.config Trusting all certificates configured for Client@32434175
>> [provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ]
>> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
>> Client@32434175[provider=null,keyStore=null,trustStore=null]
>> [junit4] 2> 5123687 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
>> [junit4] 2> 5123688 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>> [junit4] 2> 5123693 INFO
>> (zkConnectionManagerCallback-12923-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5123693 INFO (jetty-launcher-12884-thread-2) [ ]
>> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5123802 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Waiting for
>> client to connect to ZooKeeper
>> [junit4] 2> 5123803 INFO
>> (zkConnectionManagerCallback-12925-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5123803 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Client is
>> connected to ZooKeeper
>> [junit4] 2> 5123810 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ZkStateReader Updated live nodes
>> from ZooKeeper... (0) -> (2)
>> [junit4] 2> 5123813 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Publish
>> node=127.0.0.1:41191_solr as DOWN
>> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.TransientSolrCoreCacheDefault
>> Allocating transient cache for 2147483647 transient cores
>> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Register node as live
>> in ZooKeeper:/live_nodes/127.0.0.1:41191_solr
>> [junit4] 2> 5123815 INFO (zkCallback-12912-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>> [junit4] 2> 5123815 INFO (zkCallback-12900-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>> [junit4] 2> 5123817 INFO (zkCallback-12924-thread-1) [ ]
>> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>> [junit4] 2> 5123817 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.p.PackageLoader /packages.json
>> updated to version -1
>> [junit4] 2> 5123818 WARN (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.CoreContainer Not all security
>> plugins configured! authentication=disabled authorization=disabled. Solr
>> is only as secure as you make it. Consider configuring
>> authentication/authorization before exposing Solr to users internal or
>> external. See https://s.apache.org/solrsecurity for more info
>> [junit4] 2> 5123839 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
>> collection, keeping metrics history in memory.
>> [junit4] 2> 5123860 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_41191.solr.node' (registry 'solr.node') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_41191.solr.jvm' (registry 'solr.jvm') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
>> for 'solr_41191.solr.jetty' (registry 'solr.jetty') enabled at server:
>> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
>> [junit4] 2> 5123869 INFO (jetty-launcher-12884-thread-2)
>> [n:127.0.0.1:41191_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
>> definitions underneath
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node2/.
>> [junit4] 2> 5123900 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=3
>> [junit4] 2> 5123901 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5123902 INFO
>> (zkConnectionManagerCallback-12934-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5123903 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5123904 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) ->
>> (3)
>> [junit4] 2> 5123905 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at
>> 127.0.0.1:36855/solr ready
>> [junit4] 1> Target zkHost = 127.0.0.1:36855/solr
>> [junit4] 2> 5123905 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-source-001
>> [junit4] 2> 5123906 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
>> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ]
>> o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
>> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ]
>> o.a.s.c.ZkTestServer Starting server
>> [junit4] 2> 5124006 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer start zk server on port:40811
>> [junit4] 2> 5124006 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:40811
>> [junit4] 2> 5124006 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40811
>> [junit4] 2> 5124006 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40811
>> [junit4] 2> 5124009 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5124011 INFO
>> (zkConnectionManagerCallback-12936-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5124011 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5124013 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5124014 INFO
>> (zkConnectionManagerCallback-12938-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5124014 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5124015 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
>> ZooKeeper
>> [junit4] 2> 5124015 INFO
>> (zkConnectionManagerCallback-12940-thread-1) [ ]
>> o.a.s.c.c.ConnectionManager zkClient has connected
>> [junit4] 2> 5124015 INFO
>> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
>> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-3) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-2) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-1) [ ]
>> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
>> break even is size 23
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
>> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
>> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
>> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>>
>>
>> [...truncated too long message...]
>>
>> p(Native Method)
>> [junit4] 2>
>> app//com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
>> [junit4] 2>
>> [junit4] 2> 5279049 INFO
>> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
>> o.a.s.SolrTestCaseJ4
>> ------------------------------------------------------- Done waiting for
>> tracked resources to be released
>> [junit4] 2> NOTE: test params are: codec=SimpleText,
>> sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@8a39653),
>> locale=en-SD, timezone=VST
>> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation
>> 11.0.4 (64-bit)/cpus=4,threads=1,free=386818192,total=536870912
>> [junit4] 2> NOTE: All tests run in this JVM: [AuditLoggerPluginTest,
>> CdcrUpdateLogTest, SolrCloudExampleTest, ZookeeperStatusHandlerTest,
>> UUIDFieldTest, NodeLostTriggerTest,
>> TestOverriddenPrefixQueryForCustomFieldType, TestRandomFaceting,
>> TestIBSimilarityFactory, TestUtils, UpdateLogTest, TestConfigOverlay,
>> PeerSyncWithBufferUpdatesTest, TestHashQParserPlugin,
>> TestLegacyBM25SimilarityFactory, CachingDirectoryFactoryTest,
>> SortByFunctionTest, TestZkChroot, IgnoreLargeDocumentProcessorFactoryTest,
>> TestPHPSerializedResponseWriter, TestInPlaceUpdatesRequiredField,
>> TestJettySolrRunner, TestSerializedLuceneMatchVersion, DOMUtilTest,
>> CoreAdminHandlerTest, TestManagedResource, TestSolrIndexConfig,
>> BJQParserTest, TestSimLargeCluster, TestStressThreadBackup,
>> AsyncCallRequestStatusResponseTest, TestImplicitCoreProperties,
>> DistributedFacetPivotLongTailTest, WrapperMergePolicyFactoryTest,
>> TestConfigSetsAPIExclusivity, TestRTimerTree, SolrJmxReporterCloudTest,
>> QueryElevationComponentTest, NotRequiredUniqueKeyTest,
>> ChaosMonkeyNothingIsSafeTest, MBeansHandlerTest,
>> TestHighlightDedupGrouping, ZkShardTermsTest, TestJmxIntegration,
>> CurrencyRangeFacetCloudTest, ShowFileRequestHandlerTest, TestRTGBase,
>> TestCloudSearcherWarming, TestRawTransformer,
>> SignificantTermsQParserPluginTest, MinimalSchemaTest,
>> TestGeoJSONResponseWriter, AutoAddReplicasIntegrationTest,
>> TestChildDocTransformer, TestStandardQParsers, TestNestedDocsSort,
>> TestReRankQParserPlugin, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest,
>> SolrCoreCheckLockOnStartupTest, TestSSLRandomization,
>> TestLegacyNumericRangeQueryBuilder, HdfsBasicDistributedZkTest,
>> OverseerSolrResponseTest, TestPKIAuthenticationPlugin,
>> DistributedFacetPivotWhiteBoxTest, TermVectorComponentDistributedTest,
>> Tagger2Test, PeerSyncWithLeaderTest, TestLRUStatsCacheCloud,
>> ZkStateReaderTest, LeaderFailoverAfterPartitionTest,
>> MultiSolrCloudTestCaseTest, LeaderElectionContextKeyTest, TestTlogReplica,
>> SearchRateTriggerIntegrationTest, TestCollationField,
>> TestHighFrequencyDictionaryFactory, NodeAddedTriggerIntegrationTest,
>> TestConfigReload, TestSimPolicyCloud,
>> DistributedQueryComponentOptimizationTest, SolrCmdDistributorTest,
>> TestDynamicLoading, V2StandaloneTest, TestFiltering, CollectionPropsTest,
>> TestSimComputePlanAction, CloudExitableDirectoryReaderTest, PrimUtilsTest,
>> HttpPartitionOnCommitTest, ForceLeaderWithTlogReplicasTest,
>> TestSchemaManager, TestMultiValuedNumericRangeQuery,
>> TestClassicSimilarityFactory, TestBinaryResponseWriter,
>> DefaultValueUpdateProcessorTest, TestSolrCloudWithKerberosAlt,
>> CopyFieldTest, HdfsNNFailoverTest, TestManagedSynonymFilterFactory,
>> HdfsTlogReplayBufferedWhileIndexingTest, AdminHandlersProxyTest,
>> TestSolrTestCaseJ4, TestSolrConfigHandler, QueryParsingTest,
>> TestCloudJSONFacetJoinDomain, TestCollectionAPIs, TestSolrJ,
>> TestStressInPlaceUpdates, HttpTriggerListenerTest,
>> SpellCheckCollatorWithCollapseTest,
>> OverseerCollectionConfigSetProcessorTest,
>> DistributedFacetSimpleRefinementLongTailTest,
>> ScheduledMaintenanceTriggerTest, DistributedSuggestComponentTest,
>> TestPerFieldSimilarityWithDefaultOverride, TestFieldTypeCollectionResource,
>> TestExtendedDismaxParser, TestDelegationWithHadoopAuth,
>> CloudMLTQParserTest, MetricsHistoryWithAuthIntegrationTest,
>> ClassificationUpdateProcessorIntegrationTest,
>> TestLocalFSCloudBackupRestore, TestSizeLimitedDistributedMap,
>> XMLAtomicUpdateMultivalueTest, ShardSplitTest, MoveReplicaHDFSFailoverTest,
>> TestSurroundQueryParser, CoreMergeIndexesAdminHandlerTest,
>> BasicAuthStandaloneTest, TestSolrCachePerf, TestDistributedSearch,
>> StressHdfsTest, TestBulkSchemaConcurrent, TestDownShardTolerantSearch,
>> HdfsRestartWhileUpdatingTest, TestBadConfig, SolrInfoBeanTest,
>> XmlInterpolationTest, TestDistributedStatsComponentCardinality,
>> LeaderVoteWaitTimeoutTest, SearchHandlerTest, TestMacros,
>> BasicDistributedZk2Test, CollectionsAPIAsyncDistributedZkTest,
>> TestExactStatsCacheCloud, TestLeaderElectionZkExpiry,
>> TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplica,
>> TestPullReplicaErrorHandling, TestRandomRequestDistribution,
>> TestRebalanceLeaders, TestTolerantUpdateProcessorRandomCloud,
>> TestUtilizeNode, TestWaitForStateWithJettyShutdowns,
>> TriLevelCompositeIdRoutingTest, UnloadDistributedZkTest,
>> VMParamsZkACLAndCredentialsProvidersTest, ZkCLITest,
>> ConcurrentDeleteAndCreateCollectionTest,
>> TestCollectionsAPIViaSolrCloudCluster, AutoScalingHandlerTest,
>> ExecutePlanActionTest, IndexSizeTriggerSizeEstimationTest,
>> MetricTriggerIntegrationTest, SystemLogListenerTest, TestPolicyCloud,
>> TriggerCooldownIntegrationTest, TriggerSetPropertiesIntegrationTest,
>> TestSimDistributedQueue, TestSimExecutePlanAction, TestSimExtremeIndexing,
>> TestSimTriggerIntegration, TestSimUtils, TestSnapshotCloudManager,
>> BaseCdcrDistributedZkTest, CdcrBidirectionalTest, CdcrBootstrapTest]
>> [junit4] 2> NOTE: download the large Jenkins line-docs file by
>> running 'ant get-jenkins-line-docs' in the lucene directory.
>> [junit4] 2> NOTE: reproduce with: ant test
>> -Dtestcase=CdcrBootstrapTest -Dtests.seed=AE624A5368110313
>> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
>> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
>> -Dtests.locale=en-SD -Dtests.timezone=VST -Dtests.asserts=true
>> -Dtests.file.encoding=UTF-8
>> [junit4] ERROR 0.00s J1 | CdcrBootstrapTest (suite) <<<
>> [junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker
>> found 1 object(s) that were not released!!! [RawDirectoryWrapper]
>> [junit4] >
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper
>> [junit4] > at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> [junit4] > at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> [junit4] > at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> [junit4] > at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> [junit4] > at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> [junit4] > at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> [junit4] > at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> [junit4] > at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> [junit4] > at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>> [junit4] > at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> [junit4] > at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> [junit4] > at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
>> [junit4] > expected null, but was:<ObjectTracker found 1 object(s)
>> that were not released!!! [RawDirectoryWrapper]
>> [junit4] >
>> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
>> org.apache.lucene.store.RawDirectoryWrapper
>> [junit4] > at
>> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
>> [junit4] > at
>> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
>> [junit4] > at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
>> [junit4] > at
>> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
>> [junit4] > at
>> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
>> [junit4] > at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
>> [junit4] > at
>> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
>> [junit4] > at
>> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
>> [junit4] > at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>> [junit4] > at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
>> [junit4] > at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> [junit4] > at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
>> [junit4] > >
>> [junit4] > at
>> __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
>> [junit4] > at
>> org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
>> [junit4] > at
>> jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
>> [junit4] > at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> [junit4] > at
>> java.base/java.lang.reflect.Method.invoke(Method.java:566)
>> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
>> [junit4] Completed [572/900 (1!)] on J1 in 157.86s, 4 tests, 1
>> failure, 1 skipped <<< FAILURES!
>>
>> [...truncated 53511 lines...]
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: builds-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: builds-help@lucene.apache.org
>
>
>
> --
> Adrien
>
--
Adrien
Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 2096 - Still Unstable
Posted by Adrien Grand <jp...@gmail.com>.
I'm looking into it.
On Thu, Feb 6, 2020 at 7:36 AM Apache Jenkins Server <
jenkins@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2096/
>
> 3 tests failed.
> FAILED:
> org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=17673,
> name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17673, name=Thread-17208,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> at
> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> at
> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> at
> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> at
> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> at
> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> at
> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> at
> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> at
> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> at
> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> at
> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> at
> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> at
> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> at
> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> at
> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> at
> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> at
> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
>
>
> FAILED: junit.framework.TestSuite.org
> .apache.lucene.search.TestSearcherManager
>
> Error Message:
> The test or suite printed 16173 bytes to stdout and stderr, even though
> the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it
> completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
>
> Stack Trace:
> java.lang.AssertionError: The test or suite printed 16173 bytes to stdout
> and stderr, even though the limit was set to 8192 bytes. Increase the limit
> with @Limit, ignore it completely with @SuppressSysoutChecks or run with
> -Dtests.verbose=true
> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> at
> org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:282)
> at
> com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
> at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
>
> FAILED: junit.framework.TestSuite.org
> .apache.solr.cloud.cdcr.CdcrBootstrapTest
>
> Error Message:
> ObjectTracker found 1 object(s) that were not released!!!
> [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834) expected null, but
> was:<ObjectTracker found 1 object(s) that were not released!!!
> [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834) >
>
> Stack Trace:
> java.lang.AssertionError: ObjectTracker found 1 object(s) that were not
> released!!! [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper
> at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
> expected null, but was:<ObjectTracker found 1 object(s) that were not
> released!!! [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper
> at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
> >
> at __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.failNotNull(Assert.java:755)
> at org.junit.Assert.assertNull(Assert.java:737)
> at
> org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
> at jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown
> Source)
> at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
>
>
>
> Build Log:
> [...truncated 2104 lines...]
> [junit4] Suite: org.apache.lucene.search.TestSearcherManager
> [junit4] 2> Feb 06, 2020 3:18:49 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread:
> Thread[Thread-17208,5,TGRP-TestSearcherManager]
> [junit4] 2> java.lang.RuntimeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> [junit4] 2> at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> [junit4] 2> at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> [junit4] 2> at
> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> [junit4] 2> at
> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> [junit4] 2> at
> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> [junit4] 2> at
> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> [junit4] 2> at
> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> [junit4] 2> at
> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> [junit4] 2> at
> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> [junit4] 2> at
> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> [junit4] 2> at
> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> [junit4] 2> at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> [junit4] 2> at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> [junit4] 2> at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread:
> Thread[Thread-17205,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.store.AlreadyClosedException: this
> IndexWriter is closed
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
> [junit4] 2> at
> org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
> Merge Thread #6,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
> Too many open files
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
> Merge Thread #4,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
> files: this IndexWriter hit an unrecoverable exception
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by:
> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
> files: this IndexWriter hit an unrecoverable exception
> [junit4] 2> at
> org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
> [junit4] 2> at
> org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
> [junit4] 2> at
> org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> ... 3 more
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:51 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
> Merge Thread #5,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> NOTE: download the large Jenkins line-docs file by
> running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test
> -Dtestcase=TestSearcherManager
> -Dtests.method=testConcurrentIndexCloseSearchAndRefresh
> -Dtests.seed=CBAB78174F1B51A9 -Dtests.multiplier=2 -Dtests.nightly=true
> -Dtests.slow=true
> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
> -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true
> -Dtests.file.encoding=UTF-8
> [junit4] ERROR 17.7s J0 |
> TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh <<<
> [junit4] > Throwable #1:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17673, name=Thread-17208,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by: java.lang.RuntimeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> [junit4] > at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> [junit4] > at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> [junit4] > at
> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> [junit4] > at
> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> [junit4] > at
> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> [junit4] > at
> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> [junit4] > at
> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> [junit4] > at
> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> [junit4] > at
> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> [junit4] > at
> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> [junit4] > at
> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> [junit4] > at
> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> [junit4] > at
> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> [junit4] > at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> [junit4] > at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> [junit4] > at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)Throwable
> #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17688, name=Lucene Merge Thread #6,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by:
> org.apache.lucene.index.MergePolicy$MergeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
> Too many open files
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable
> #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17670, name=Thread-17205,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by:
> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
> [junit4] > at
> org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable
> #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17686, name=Lucene Merge Thread #4,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by:
> org.apache.lucene.index.MergePolicy$MergeException:
> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
> files: this IndexWriter hit an unrecoverable exception
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by:
> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
> files: this IndexWriter hit an unrecoverable exception
> [junit4] > at
> org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
> [junit4] > at
> org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
> [junit4] > at
> org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > ... 3 moreThrowable #5:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17687, name=Lucene Merge Thread #5,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by:
> org.apache.lucene.index.MergePolicy$MergeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2> NOTE: test params are: codec=Asserting(Lucene84),
> sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@277d98a5),
> locale=en-US, timezone=Pacific/Tarawa
> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation
> 11.0.4 (64-bit)/cpus=4,threads=1,free=244326912,total=536870912
> [junit4] 2> NOTE: All tests run in this JVM:
> [TestSameScoresWithThreads, TestLucene84PostingsFormat,
> TestRamUsageEstimator, TestLatLonMultiLineShapeQueries,
> TestEarlyTermination, TestSimpleFSLockFactory, TestMatchesIterator,
> TestStressIndexing, TestMultiFields, TestIndexCommit,
> TestDisjunctionScoreBlockBoundaryPropagator, TestMixedCodecs,
> TestLucene50CompoundFormat, TestField, TestTotalHits, TestIndexedDISI,
> TestSegmentCacheables, TestRegExp, TestDocIdsWriter, TestFieldReuse,
> TestIntBlockPool, TestIndexWriterThreadsToSegments, TestReaderPool,
> TestNoMergePolicy, TestPerFieldPostingsFormat, MultiCollectorTest,
> TestComplexExplanationsOfNonMatches, TestLogMergePolicy, TestTermStates,
> TestPendingDeletes, TestFeatureDoubleValues, TestMultiset,
> TestSortRescorer, TestRollingUpdates, TestSimilarityBase,
> TestTermVectorsWriter, TestXYRectangle2D, TestAddIndexes, TestTransactions,
> TestSpansEnum, TestVirtualMethod, TestIndexManyDocuments, TestGeoUtils,
> TestLatLonPointShapeQueries, TestConstantScoreQuery, TestWordlistLoader,
> TestIntsRef, TestIndexSorting, TestMaxScoreSumPropagator, TestNeverDelete,
> TestSegmentTermEnum, TestLucene60FieldInfoFormat, TestXYLineShapeQueries,
> TestFieldValueQuery, TestPrefixCodedTerms, TestBM25Similarity,
> TestAxiomaticF1LOG, TestFixedBitDocIdSet, TestUniqueTermCount,
> TestFilterMergePolicy, TestTimeLimitingCollector, TestPhraseQuery,
> TestMaxPosition, TestSimpleExplanationsWithFillerDocs, TestSentinelIntSet,
> TestDocValuesRewriteMethod, TestDaciukMihovAutomatonBuilder,
> TestNRTReaderCleanup, TestCustomNorms, TestSnapshotDeletionPolicy,
> TestDuelingCodecs, TestStringHelper, TestIntRange,
> TestAllFilesHaveChecksumFooter, TestCharsRef, TestPriorityQueue,
> TestLevenshteinAutomata, TestFuzzyQuery, TestXYShape, TestBinaryTerms,
> TestNumericUtils, TestPrefixRandom, TestAutomatonQuery,
> TestTimSorterWorstCase, TestIndexWriterOnVMError,
> TestXYMultiPolygonShapeQueries, TestDirectMonotonic, TestLatLonPoint,
> TestLongValuesSource, TestNativeFSLockFactory, TestIntRangeFieldQueries,
> TestSortedSetSortField, TestLucene50StoredFieldsFormatMergeInstance,
> TestLucene80NormsFormatMergeInstance, Test2BPostingsBytes,
> Test2BSortedDocValuesFixedSorted, Test2BSortedDocValuesOrds, Test2BTerms,
> Test4GBStoredFields, TestBagOfPositions, TestBagOfPostings,
> TestBinaryDocValuesUpdates, TestCodecHoldsOpenFiles, TestCodecs,
> TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash,
> TestCrashCausesCorruptIndex, TestDemoParallelLeafReader,
> TestExitableDirectoryReader, TestFieldInfos, TestFieldInvertState,
> TestFilterCodecReader, TestFilterDirectoryReader, TestFilterLeafReader,
> TestForTooMuchCloning, TestForceMergeForever, TestFrozenBufferedUpdates,
> TestIndexWriterCommit, TestIndexWriterConfig, TestIndexWriterDelete,
> TestIndexWriterDeleteByQuery, TestIndexWriterExceptions,
> TestIndexWriterOnDiskFull, TestOmitNorms, TestOmitPositions, TestOmitTf,
> TestOneMergeWrappingMergePolicy, TestOrdinalMap,
> TestParallelCompositeReader, TestParallelLeafReader,
> TestParallelReaderEmptyIndex, TestParallelTermEnum, TestPayloads,
> TestPayloadsOnVectors, TestPersistentSnapshotDeletionPolicy,
> TestPointValues, TestPostingsOffsets, TestSegmentMerger, TestSegmentReader,
> TestSegmentTermDocs, TestSortedSetDocValues, TestStressAdvance,
> TestStressDeletes, TestSumDocFreq, TestSwappedIndexFiles, TestTermsEnum2,
> TestThreadedForceMerge, TestTieredMergePolicy,
> TestTragicIndexWriterDeadlock, TestTransactionRollback,
> TestBlockMaxConjunction, TestBoolean2, TestBoolean2ScorerSupplier,
> TestBooleanMinShouldMatch, TestBooleanOr, TestBooleanQuery,
> TestBooleanQueryVisitSubscorers, TestBooleanRewrites, TestBooleanScorer,
> TestBoostQuery, TestCachingCollector, TestComplexExplanations,
> TestCustomSearcherSort, TestDateSort, TestDisjunctionMaxQuery,
> TestLatLonPointQueries, TestLiveFieldValues, TestLongRangeFieldQueries,
> TestSearcherManager]
> [junit4] 2> NOTE: download the large Jenkins line-docs file by
> running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test
> -Dtestcase=TestSearcherManager -Dtests.seed=CBAB78174F1B51A9
> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
> -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true
> -Dtests.file.encoding=UTF-8
> [junit4] ERROR 0.00s J0 | TestSearcherManager (suite) <<<
> [junit4] > Throwable #1: java.lang.AssertionError: The test or suite
> printed 16173 bytes to stdout and stderr, even though the limit was set to
> 8192 bytes. Increase the limit with @Limit, ignore it completely with
> @SuppressSysoutChecks or run with -Dtests.verbose=true
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] Completed [546/552 (1!)] on J0 in 355.57s, 11 tests, 1
> failure, 1 error <<< FAILURES!
>
> [...truncated 12649 lines...]
> [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBootstrapTest
> [junit4] 2> 5121336 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 Created dataDir:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/data-dir-186-001
> [junit4] 2> 5121337 WARN
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11
> [junit4] 2> 5121337 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
> w/NUMERIC_DOCVALUES_SYSPROP=false
> [junit4] 2> 5121337 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via:
> @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0,
> clientAuth=0.0/0.0)
> [junit4] 2> 5121338 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 SecureRandom sanity checks:
> test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
> [junit4] 2> 5121341 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithMultipleReplicas
> [junit4] 2> 5121341 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001
> [junit4] 2> 5121341 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ]
> o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ]
> o.a.s.c.ZkTestServer Starting server
> [junit4] 2> 5121442 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer start zk server on port:36855
> [junit4] 2> 5121442 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:36855
> [junit4] 2> 5121442 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36855
> [junit4] 2> 5121442 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36855
> [junit4] 2> 5121443 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5121452 INFO
> (zkConnectionManagerCallback-12879-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121452 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121454 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5121455 INFO
> (zkConnectionManagerCallback-12881-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121455 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121456 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5121457 INFO
> (zkConnectionManagerCallback-12883-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121457 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.session node0 Scavenging every 600000ms
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@867eda0
> {/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.session node0 Scavenging every 660000ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.session node0 Scavenging every 660000ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d1dd263
> {/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ecf5186
> {/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.AbstractConnector Started ServerConnector@3473a5ca{ssl,[ssl,
> alpn, http/1.1, h2]}{127.0.0.1:41191}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.Server Started @5121593ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
> hostPort=41191}
> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-2) [ ]
> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
> be missing or incomplete.
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter Using logger factory
> org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.AbstractConnector Started ServerConnector@77d77ae2{ssl,[ssl,
> alpn, http/1.1, h2]}{127.0.0.1:37757}
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.Server Started @5121593ms
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
> hostPort=37757}
> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-1) [ ]
> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
> be missing or incomplete.
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter Using logger factory
> org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
> version 9.0.0
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
> version 9.0.0
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
> null
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
> null
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
> 2020-02-06T05:48:19.821958Z
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
> 2020-02-06T05:48:19.821957Z
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.AbstractConnector Started ServerConnector@16db638b{ssl,[ssl,
> alpn, http/1.1, h2]}{127.0.0.1:43461}
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.Server Started @5121594ms
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
> hostPort=43461}
> [junit4] 2> 5121567 ERROR (jetty-launcher-12884-thread-3) [ ]
> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
> be missing or incomplete.
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter Using logger factory
> org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
> version 9.0.0
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
> null
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
> 2020-02-06T05:48:19.822974Z
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121569 INFO
> (zkConnectionManagerCallback-12886-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121569 INFO
> (zkConnectionManagerCallback-12888-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121569 INFO
> (zkConnectionManagerCallback-12890-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5122191 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
> WhitelistHostChecker [whitelistHosts=null,
> whitelistHostCheckingEnabled=true]
> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@9c25133
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@9c25133[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@3dc1efed
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@3dc1efed[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122224 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5122229 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5122230 INFO
> (zkConnectionManagerCallback-12899-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122230 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5122332 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Waiting for
> client to connect to ZooKeeper
> [junit4] 2> 5122333 INFO
> (zkConnectionManagerCallback-12901-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122333 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Client is
> connected to ZooKeeper
> [junit4] 2> 5122450 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.OverseerElectionContext I am going
> to be the leader 127.0.0.1:37757_solr
> [junit4] 2> 5122451 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Overseer
> (id=73197244948086791-127.0.0.1:37757_solr-n_0000000000) starting
> [junit4] 2> 5122454 INFO
> (OverseerStateUpdate-73197244948086791-127.0.0.1:37757_solr-n_0000000000)
> [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Starting to work on the
> main queue : 127.0.0.1:37757_solr
> [junit4] 2> 5122464 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.ZkController Register node as live
> in ZooKeeper:/live_nodes/127.0.0.1:37757_solr
> [junit4] 2> 5122467 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.p.PackageLoader /packages.json
> updated to version -1
> [junit4] 2> 5122467 WARN (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.CoreContainer Not all security
> plugins configured! authentication=disabled authorization=disabled. Solr
> is only as secure as you make it. Consider configuring
> authentication/authorization before exposing Solr to users internal or
> external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5122475 INFO (zkCallback-12900-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
> [junit4] 2> 5122500 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
> collection, keeping metrics history in memory.
> [junit4] 2> 5122523 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_37757.solr.node' (registry 'solr.node') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_37757.solr.jvm' (registry 'solr.jvm') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_37757.solr.jetty' (registry 'solr.jetty') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122533 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
> definitions underneath
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node1/.
> [junit4] 2> 5122737 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
> WhitelistHostChecker [whitelistHosts=null,
> whitelistHostCheckingEnabled=true]
> [junit4] 2> 5122738 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@2d6e218d
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122739 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@2d6e218d[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@1ed7f896
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@1ed7f896[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122743 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5122744 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5122746 INFO
> (zkConnectionManagerCallback-12911-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122746 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5122848 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Waiting for
> client to connect to ZooKeeper
> [junit4] 2> 5122852 INFO
> (zkConnectionManagerCallback-12913-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122852 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Client is
> connected to ZooKeeper
> [junit4] 2> 5122864 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ZkStateReader Updated live nodes
> from ZooKeeper... (0) -> (1)
> [junit4] 2> 5122868 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Publish
> node=127.0.0.1:43461_solr as DOWN
> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.TransientSolrCoreCacheDefault
> Allocating transient cache for 2147483647 transient cores
> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Register node as live
> in ZooKeeper:/live_nodes/127.0.0.1:43461_solr
> [junit4] 2> 5122870 INFO (zkCallback-12900-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
> [junit4] 2> 5122870 INFO (zkCallback-12912-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
> [junit4] 2> 5122871 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.p.PackageLoader /packages.json
> updated to version -1
> [junit4] 2> 5122871 WARN (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.CoreContainer Not all security
> plugins configured! authentication=disabled authorization=disabled. Solr
> is only as secure as you make it. Consider configuring
> authentication/authorization before exposing Solr to users internal or
> external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5122894 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
> collection, keeping metrics history in memory.
> [junit4] 2> 5122918 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_43461.solr.node' (registry 'solr.node') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122929 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_43461.solr.jvm' (registry 'solr.jvm') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122930 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_43461.solr.jetty' (registry 'solr.jetty') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122931 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
> definitions underneath
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node3/.
> [junit4] 2> 5123682 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
> WhitelistHostChecker [whitelistHosts=null,
> whitelistHostCheckingEnabled=true]
> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@32f87da4
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@32f87da4[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@32434175
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@32434175[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123687 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5123688 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5123693 INFO
> (zkConnectionManagerCallback-12923-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123693 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5123802 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Waiting for
> client to connect to ZooKeeper
> [junit4] 2> 5123803 INFO
> (zkConnectionManagerCallback-12925-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123803 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Client is
> connected to ZooKeeper
> [junit4] 2> 5123810 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ZkStateReader Updated live nodes
> from ZooKeeper... (0) -> (2)
> [junit4] 2> 5123813 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Publish
> node=127.0.0.1:41191_solr as DOWN
> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.TransientSolrCoreCacheDefault
> Allocating transient cache for 2147483647 transient cores
> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Register node as live
> in ZooKeeper:/live_nodes/127.0.0.1:41191_solr
> [junit4] 2> 5123815 INFO (zkCallback-12912-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123815 INFO (zkCallback-12900-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123817 INFO (zkCallback-12924-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123817 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.p.PackageLoader /packages.json
> updated to version -1
> [junit4] 2> 5123818 WARN (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.CoreContainer Not all security
> plugins configured! authentication=disabled authorization=disabled. Solr
> is only as secure as you make it. Consider configuring
> authentication/authorization before exposing Solr to users internal or
> external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5123839 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
> collection, keeping metrics history in memory.
> [junit4] 2> 5123860 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_41191.solr.node' (registry 'solr.node') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_41191.solr.jvm' (registry 'solr.jvm') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_41191.solr.jetty' (registry 'solr.jetty') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123869 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
> definitions underneath
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node2/.
> [junit4] 2> 5123900 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=3
> [junit4] 2> 5123901 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5123902 INFO
> (zkConnectionManagerCallback-12934-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123903 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5123904 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) ->
> (3)
> [junit4] 2> 5123905 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at
> 127.0.0.1:36855/solr ready
> [junit4] 1> Target zkHost = 127.0.0.1:36855/solr
> [junit4] 2> 5123905 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-source-001
> [junit4] 2> 5123906 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ]
> o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ]
> o.a.s.c.ZkTestServer Starting server
> [junit4] 2> 5124006 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer start zk server on port:40811
> [junit4] 2> 5124006 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:40811
> [junit4] 2> 5124006 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40811
> [junit4] 2> 5124006 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40811
> [junit4] 2> 5124009 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5124011 INFO
> (zkConnectionManagerCallback-12936-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124011 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124013 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5124014 INFO
> (zkConnectionManagerCallback-12938-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124014 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124015 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5124015 INFO
> (zkConnectionManagerCallback-12940-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124015 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-3) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-2) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-1) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>
>
> [...truncated too long message...]
>
> p(Native Method)
> [junit4] 2>
> app//com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
> [junit4] 2>
> [junit4] 2> 5279049 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4
> ------------------------------------------------------- Done waiting for
> tracked resources to be released
> [junit4] 2> NOTE: test params are: codec=SimpleText,
> sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@8a39653),
> locale=en-SD, timezone=VST
> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation
> 11.0.4 (64-bit)/cpus=4,threads=1,free=386818192,total=536870912
> [junit4] 2> NOTE: All tests run in this JVM: [AuditLoggerPluginTest,
> CdcrUpdateLogTest, SolrCloudExampleTest, ZookeeperStatusHandlerTest,
> UUIDFieldTest, NodeLostTriggerTest,
> TestOverriddenPrefixQueryForCustomFieldType, TestRandomFaceting,
> TestIBSimilarityFactory, TestUtils, UpdateLogTest, TestConfigOverlay,
> PeerSyncWithBufferUpdatesTest, TestHashQParserPlugin,
> TestLegacyBM25SimilarityFactory, CachingDirectoryFactoryTest,
> SortByFunctionTest, TestZkChroot, IgnoreLargeDocumentProcessorFactoryTest,
> TestPHPSerializedResponseWriter, TestInPlaceUpdatesRequiredField,
> TestJettySolrRunner, TestSerializedLuceneMatchVersion, DOMUtilTest,
> CoreAdminHandlerTest, TestManagedResource, TestSolrIndexConfig,
> BJQParserTest, TestSimLargeCluster, TestStressThreadBackup,
> AsyncCallRequestStatusResponseTest, TestImplicitCoreProperties,
> DistributedFacetPivotLongTailTest, WrapperMergePolicyFactoryTest,
> TestConfigSetsAPIExclusivity, TestRTimerTree, SolrJmxReporterCloudTest,
> QueryElevationComponentTest, NotRequiredUniqueKeyTest,
> ChaosMonkeyNothingIsSafeTest, MBeansHandlerTest,
> TestHighlightDedupGrouping, ZkShardTermsTest, TestJmxIntegration,
> CurrencyRangeFacetCloudTest, ShowFileRequestHandlerTest, TestRTGBase,
> TestCloudSearcherWarming, TestRawTransformer,
> SignificantTermsQParserPluginTest, MinimalSchemaTest,
> TestGeoJSONResponseWriter, AutoAddReplicasIntegrationTest,
> TestChildDocTransformer, TestStandardQParsers, TestNestedDocsSort,
> TestReRankQParserPlugin, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest,
> SolrCoreCheckLockOnStartupTest, TestSSLRandomization,
> TestLegacyNumericRangeQueryBuilder, HdfsBasicDistributedZkTest,
> OverseerSolrResponseTest, TestPKIAuthenticationPlugin,
> DistributedFacetPivotWhiteBoxTest, TermVectorComponentDistributedTest,
> Tagger2Test, PeerSyncWithLeaderTest, TestLRUStatsCacheCloud,
> ZkStateReaderTest, LeaderFailoverAfterPartitionTest,
> MultiSolrCloudTestCaseTest, LeaderElectionContextKeyTest, TestTlogReplica,
> SearchRateTriggerIntegrationTest, TestCollationField,
> TestHighFrequencyDictionaryFactory, NodeAddedTriggerIntegrationTest,
> TestConfigReload, TestSimPolicyCloud,
> DistributedQueryComponentOptimizationTest, SolrCmdDistributorTest,
> TestDynamicLoading, V2StandaloneTest, TestFiltering, CollectionPropsTest,
> TestSimComputePlanAction, CloudExitableDirectoryReaderTest, PrimUtilsTest,
> HttpPartitionOnCommitTest, ForceLeaderWithTlogReplicasTest,
> TestSchemaManager, TestMultiValuedNumericRangeQuery,
> TestClassicSimilarityFactory, TestBinaryResponseWriter,
> DefaultValueUpdateProcessorTest, TestSolrCloudWithKerberosAlt,
> CopyFieldTest, HdfsNNFailoverTest, TestManagedSynonymFilterFactory,
> HdfsTlogReplayBufferedWhileIndexingTest, AdminHandlersProxyTest,
> TestSolrTestCaseJ4, TestSolrConfigHandler, QueryParsingTest,
> TestCloudJSONFacetJoinDomain, TestCollectionAPIs, TestSolrJ,
> TestStressInPlaceUpdates, HttpTriggerListenerTest,
> SpellCheckCollatorWithCollapseTest,
> OverseerCollectionConfigSetProcessorTest,
> DistributedFacetSimpleRefinementLongTailTest,
> ScheduledMaintenanceTriggerTest, DistributedSuggestComponentTest,
> TestPerFieldSimilarityWithDefaultOverride, TestFieldTypeCollectionResource,
> TestExtendedDismaxParser, TestDelegationWithHadoopAuth,
> CloudMLTQParserTest, MetricsHistoryWithAuthIntegrationTest,
> ClassificationUpdateProcessorIntegrationTest,
> TestLocalFSCloudBackupRestore, TestSizeLimitedDistributedMap,
> XMLAtomicUpdateMultivalueTest, ShardSplitTest, MoveReplicaHDFSFailoverTest,
> TestSurroundQueryParser, CoreMergeIndexesAdminHandlerTest,
> BasicAuthStandaloneTest, TestSolrCachePerf, TestDistributedSearch,
> StressHdfsTest, TestBulkSchemaConcurrent, TestDownShardTolerantSearch,
> HdfsRestartWhileUpdatingTest, TestBadConfig, SolrInfoBeanTest,
> XmlInterpolationTest, TestDistributedStatsComponentCardinality,
> LeaderVoteWaitTimeoutTest, SearchHandlerTest, TestMacros,
> BasicDistributedZk2Test, CollectionsAPIAsyncDistributedZkTest,
> TestExactStatsCacheCloud, TestLeaderElectionZkExpiry,
> TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplica,
> TestPullReplicaErrorHandling, TestRandomRequestDistribution,
> TestRebalanceLeaders, TestTolerantUpdateProcessorRandomCloud,
> TestUtilizeNode, TestWaitForStateWithJettyShutdowns,
> TriLevelCompositeIdRoutingTest, UnloadDistributedZkTest,
> VMParamsZkACLAndCredentialsProvidersTest, ZkCLITest,
> ConcurrentDeleteAndCreateCollectionTest,
> TestCollectionsAPIViaSolrCloudCluster, AutoScalingHandlerTest,
> ExecutePlanActionTest, IndexSizeTriggerSizeEstimationTest,
> MetricTriggerIntegrationTest, SystemLogListenerTest, TestPolicyCloud,
> TriggerCooldownIntegrationTest, TriggerSetPropertiesIntegrationTest,
> TestSimDistributedQueue, TestSimExecutePlanAction, TestSimExtremeIndexing,
> TestSimTriggerIntegration, TestSimUtils, TestSnapshotCloudManager,
> BaseCdcrDistributedZkTest, CdcrBidirectionalTest, CdcrBootstrapTest]
> [junit4] 2> NOTE: download the large Jenkins line-docs file by
> running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test
> -Dtestcase=CdcrBootstrapTest -Dtests.seed=AE624A5368110313
> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
> -Dtests.locale=en-SD -Dtests.timezone=VST -Dtests.asserts=true
> -Dtests.file.encoding=UTF-8
> [junit4] ERROR 0.00s J1 | CdcrBootstrapTest (suite) <<<
> [junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker
> found 1 object(s) that were not released!!! [RawDirectoryWrapper]
> [junit4] >
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper
> [junit4] > at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> [junit4] > at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> [junit4] > at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> [junit4] > at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> [junit4] > at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> [junit4] > at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> [junit4] > at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> [junit4] > at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> [junit4] > at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> [junit4] > at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> [junit4] > at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [junit4] > at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] > expected null, but was:<ObjectTracker found 1 object(s)
> that were not released!!! [RawDirectoryWrapper]
> [junit4] >
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper
> [junit4] > at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> [junit4] > at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> [junit4] > at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> [junit4] > at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> [junit4] > at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> [junit4] > at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> [junit4] > at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> [junit4] > at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> [junit4] > at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> [junit4] > at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> [junit4] > at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [junit4] > at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] > >
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
> [junit4] > at
> org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
> [junit4] > at
> jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
> [junit4] > at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [junit4] > at
> java.base/java.lang.reflect.Method.invoke(Method.java:566)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] Completed [572/900 (1!)] on J1 in 157.86s, 4 tests, 1 failure,
> 1 skipped <<< FAILURES!
>
> [...truncated 53511 lines...]
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: builds-unsubscribe@lucene.apache.org
> For additional commands, e-mail: builds-help@lucene.apache.org
--
Adrien
Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 2096 - Still Unstable
Posted by Adrien Grand <jp...@gmail.com>.
I'm looking into it.
On Thu, Feb 6, 2020 at 7:36 AM Apache Jenkins Server <
jenkins@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2096/
>
> 3 tests failed.
> FAILED:
> org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=17673,
> name=Thread-17208, state=RUNNABLE, group=TGRP-TestSearcherManager]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17673, name=Thread-17208,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> at
> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> at
> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> at
> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> at
> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> at
> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> at
> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> at
> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> at
> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> at
> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> at
> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> at
> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> at
> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> at
> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> at
> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> at
> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> at
> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
>
>
> FAILED: junit.framework.TestSuite.org
> .apache.lucene.search.TestSearcherManager
>
> Error Message:
> The test or suite printed 16173 bytes to stdout and stderr, even though
> the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it
> completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
>
> Stack Trace:
> java.lang.AssertionError: The test or suite printed 16173 bytes to stdout
> and stderr, even though the limit was set to 8192 bytes. Increase the limit
> with @Limit, ignore it completely with @SuppressSysoutChecks or run with
> -Dtests.verbose=true
> at __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> at
> org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:282)
> at
> com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
> at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
>
> FAILED: junit.framework.TestSuite.org
> .apache.solr.cloud.cdcr.CdcrBootstrapTest
>
> Error Message:
> ObjectTracker found 1 object(s) that were not released!!!
> [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834) expected null, but
> was:<ObjectTracker found 1 object(s) that were not released!!!
> [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834) >
>
> Stack Trace:
> java.lang.AssertionError: ObjectTracker found 1 object(s) that were not
> released!!! [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper
> at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
> expected null, but was:<ObjectTracker found 1 object(s) that were not
> released!!! [RawDirectoryWrapper]
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper
> at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
> >
> at __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.failNotNull(Assert.java:755)
> at org.junit.Assert.assertNull(Assert.java:737)
> at
> org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
> at jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown
> Source)
> at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
> at java.base/java.lang.Thread.run(Thread.java:834)
>
>
>
>
> Build Log:
> [...truncated 2104 lines...]
> [junit4] Suite: org.apache.lucene.search.TestSearcherManager
> [junit4] 2> Feb 06, 2020 3:18:49 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread:
> Thread[Thread-17208,5,TGRP-TestSearcherManager]
> [junit4] 2> java.lang.RuntimeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> [junit4] 2> at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> [junit4] 2> at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> [junit4] 2> at
> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> [junit4] 2> at
> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> [junit4] 2> at
> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> [junit4] 2> at
> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> [junit4] 2> at
> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> [junit4] 2> at
> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> [junit4] 2> at
> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> [junit4] 2> at
> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> [junit4] 2> at
> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> [junit4] 2> at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> [junit4] 2> at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> [junit4] 2> at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread:
> Thread[Thread-17205,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.store.AlreadyClosedException: this
> IndexWriter is closed
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
> [junit4] 2> at
> org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
> Merge Thread #6,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
> Too many open files
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:50 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
> Merge Thread #4,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
> files: this IndexWriter hit an unrecoverable exception
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by:
> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
> files: this IndexWriter hit an unrecoverable exception
> [junit4] 2> at
> org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
> [junit4] 2> at
> org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
> [junit4] 2> at
> org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> ... 3 more
> [junit4] 2>
> [junit4] 2> Feb 06, 2020 3:18:51 PM
> com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
> uncaughtException
> [junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene
> Merge Thread #5,5,TGRP-TestSearcherManager]
> [junit4] 2> org.apache.lucene.index.MergePolicy$MergeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] 2> at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] 2> Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] 2> at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] 2> at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] 2> at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] 2> at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] 2> at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] 2> at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] 2> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] 2> at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] 2> at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] 2> at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] 2> at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] 2> at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2>
> [junit4] 2> NOTE: download the large Jenkins line-docs file by
> running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test
> -Dtestcase=TestSearcherManager
> -Dtests.method=testConcurrentIndexCloseSearchAndRefresh
> -Dtests.seed=CBAB78174F1B51A9 -Dtests.multiplier=2 -Dtests.nightly=true
> -Dtests.slow=true
> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
> -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true
> -Dtests.file.encoding=UTF-8
> [junit4] ERROR 17.7s J0 |
> TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh <<<
> [junit4] > Throwable #1:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17673, name=Thread-17208,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by: java.lang.RuntimeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:678)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_dp.nvm:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:197)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
> [junit4] > at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
> [junit4] > at
> java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
> [junit4] > at
> org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81)
> [junit4] > at
> org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2890)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:742)
> [junit4] > at
> org.apache.lucene.store.Directory.openChecksumInput(Directory.java:157)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1038)
> [junit4] > at
> org.apache.lucene.codecs.lucene80.Lucene80NormsProducer.<init>(Lucene80NormsProducer.java:59)
> [junit4] > at
> org.apache.lucene.codecs.lucene80.Lucene80NormsFormat.normsProducer(Lucene80NormsFormat.java:90)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingNormsFormat.normsProducer(AssertingNormsFormat.java:51)
> [junit4] > at
> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:121)
> [junit4] > at
> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
> [junit4] > at
> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
> [junit4] > at
> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
> [junit4] > at
> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
> [junit4] > at
> org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
> [junit4] > at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
> [junit4] > at
> org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
> [junit4] > at
> org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:666)Throwable
> #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17688, name=Lucene Merge Thread #6,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by:
> org.apache.lucene.index.MergePolicy$MergeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
> Too many open files
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f2_doc_ids_12x.tmp:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable
> #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17670, name=Thread-17205,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by:
> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:681)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:695)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1591)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
> [junit4] > at
> org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:576)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)Throwable
> #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17686, name=Lucene Merge Thread #4,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by:
> org.apache.lucene.index.MergePolicy$MergeException:
> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
> files: this IndexWriter hit an unrecoverable exception
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by:
> org.apache.lucene.store.AlreadyClosedException: refusing to delete any
> files: this IndexWriter hit an unrecoverable exception
> [junit4] > at
> org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
> [junit4] > at
> org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
> [junit4] > at
> org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5017)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4545)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > ... 3 moreThrowable #5:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=17687, name=Lucene Merge Thread #5,
> state=RUNNABLE, group=TGRP-TestSearcherManager]
> [junit4] > Caused by:
> org.apache.lucene.index.MergePolicy$MergeException:
> java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:704)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
> [junit4] > Caused by: java.nio.file.FileSystemException:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.search.TestSearcherManager_CBAB78174F1B51A9-001/tempDir-001/_f1_doc_ids_12w.tmp:
> Too many open files
> [junit4] > at
> org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> [junit4] > at
> org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:160)
> [junit4] > at
> org.apache.lucene.mockfile.FilterFileSystemProvider.newOutputStream(FilterFileSystemProvider.java:197)
> [junit4] > at
> java.base/java.nio.file.Files.newOutputStream(Files.java:219)
> [junit4] > at
> org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410)
> [junit4] > at
> org.apache.lucene.store.FSDirectory.createTempOutput(FSDirectory.java:267)
> [junit4] > at
> org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:697)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.FilterDirectory.createTempOutput(FilterDirectory.java:79)
> [junit4] > at
> org.apache.lucene.store.TrackingDirectoryWrapper.createTempOutput(TrackingDirectoryWrapper.java:51)
> [junit4] > at
> org.apache.lucene.codecs.compressing.FieldsIndexWriter.<init>(FieldsIndexWriter.java:82)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:117)
> [junit4] > at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130)
> [junit4] > at
> org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:145)
> [junit4] > at
> org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:48)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
> [junit4] > at
> org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4446)
> [junit4] > at
> org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4040)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
> [junit4] > at
> org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
> [junit4] 2> NOTE: test params are: codec=Asserting(Lucene84),
> sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@277d98a5),
> locale=en-US, timezone=Pacific/Tarawa
> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation
> 11.0.4 (64-bit)/cpus=4,threads=1,free=244326912,total=536870912
> [junit4] 2> NOTE: All tests run in this JVM:
> [TestSameScoresWithThreads, TestLucene84PostingsFormat,
> TestRamUsageEstimator, TestLatLonMultiLineShapeQueries,
> TestEarlyTermination, TestSimpleFSLockFactory, TestMatchesIterator,
> TestStressIndexing, TestMultiFields, TestIndexCommit,
> TestDisjunctionScoreBlockBoundaryPropagator, TestMixedCodecs,
> TestLucene50CompoundFormat, TestField, TestTotalHits, TestIndexedDISI,
> TestSegmentCacheables, TestRegExp, TestDocIdsWriter, TestFieldReuse,
> TestIntBlockPool, TestIndexWriterThreadsToSegments, TestReaderPool,
> TestNoMergePolicy, TestPerFieldPostingsFormat, MultiCollectorTest,
> TestComplexExplanationsOfNonMatches, TestLogMergePolicy, TestTermStates,
> TestPendingDeletes, TestFeatureDoubleValues, TestMultiset,
> TestSortRescorer, TestRollingUpdates, TestSimilarityBase,
> TestTermVectorsWriter, TestXYRectangle2D, TestAddIndexes, TestTransactions,
> TestSpansEnum, TestVirtualMethod, TestIndexManyDocuments, TestGeoUtils,
> TestLatLonPointShapeQueries, TestConstantScoreQuery, TestWordlistLoader,
> TestIntsRef, TestIndexSorting, TestMaxScoreSumPropagator, TestNeverDelete,
> TestSegmentTermEnum, TestLucene60FieldInfoFormat, TestXYLineShapeQueries,
> TestFieldValueQuery, TestPrefixCodedTerms, TestBM25Similarity,
> TestAxiomaticF1LOG, TestFixedBitDocIdSet, TestUniqueTermCount,
> TestFilterMergePolicy, TestTimeLimitingCollector, TestPhraseQuery,
> TestMaxPosition, TestSimpleExplanationsWithFillerDocs, TestSentinelIntSet,
> TestDocValuesRewriteMethod, TestDaciukMihovAutomatonBuilder,
> TestNRTReaderCleanup, TestCustomNorms, TestSnapshotDeletionPolicy,
> TestDuelingCodecs, TestStringHelper, TestIntRange,
> TestAllFilesHaveChecksumFooter, TestCharsRef, TestPriorityQueue,
> TestLevenshteinAutomata, TestFuzzyQuery, TestXYShape, TestBinaryTerms,
> TestNumericUtils, TestPrefixRandom, TestAutomatonQuery,
> TestTimSorterWorstCase, TestIndexWriterOnVMError,
> TestXYMultiPolygonShapeQueries, TestDirectMonotonic, TestLatLonPoint,
> TestLongValuesSource, TestNativeFSLockFactory, TestIntRangeFieldQueries,
> TestSortedSetSortField, TestLucene50StoredFieldsFormatMergeInstance,
> TestLucene80NormsFormatMergeInstance, Test2BPostingsBytes,
> Test2BSortedDocValuesFixedSorted, Test2BSortedDocValuesOrds, Test2BTerms,
> Test4GBStoredFields, TestBagOfPositions, TestBagOfPostings,
> TestBinaryDocValuesUpdates, TestCodecHoldsOpenFiles, TestCodecs,
> TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash,
> TestCrashCausesCorruptIndex, TestDemoParallelLeafReader,
> TestExitableDirectoryReader, TestFieldInfos, TestFieldInvertState,
> TestFilterCodecReader, TestFilterDirectoryReader, TestFilterLeafReader,
> TestForTooMuchCloning, TestForceMergeForever, TestFrozenBufferedUpdates,
> TestIndexWriterCommit, TestIndexWriterConfig, TestIndexWriterDelete,
> TestIndexWriterDeleteByQuery, TestIndexWriterExceptions,
> TestIndexWriterOnDiskFull, TestOmitNorms, TestOmitPositions, TestOmitTf,
> TestOneMergeWrappingMergePolicy, TestOrdinalMap,
> TestParallelCompositeReader, TestParallelLeafReader,
> TestParallelReaderEmptyIndex, TestParallelTermEnum, TestPayloads,
> TestPayloadsOnVectors, TestPersistentSnapshotDeletionPolicy,
> TestPointValues, TestPostingsOffsets, TestSegmentMerger, TestSegmentReader,
> TestSegmentTermDocs, TestSortedSetDocValues, TestStressAdvance,
> TestStressDeletes, TestSumDocFreq, TestSwappedIndexFiles, TestTermsEnum2,
> TestThreadedForceMerge, TestTieredMergePolicy,
> TestTragicIndexWriterDeadlock, TestTransactionRollback,
> TestBlockMaxConjunction, TestBoolean2, TestBoolean2ScorerSupplier,
> TestBooleanMinShouldMatch, TestBooleanOr, TestBooleanQuery,
> TestBooleanQueryVisitSubscorers, TestBooleanRewrites, TestBooleanScorer,
> TestBoostQuery, TestCachingCollector, TestComplexExplanations,
> TestCustomSearcherSort, TestDateSort, TestDisjunctionMaxQuery,
> TestLatLonPointQueries, TestLiveFieldValues, TestLongRangeFieldQueries,
> TestSearcherManager]
> [junit4] 2> NOTE: download the large Jenkins line-docs file by
> running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test
> -Dtestcase=TestSearcherManager -Dtests.seed=CBAB78174F1B51A9
> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
> -Dtests.locale=en-US -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true
> -Dtests.file.encoding=UTF-8
> [junit4] ERROR 0.00s J0 | TestSearcherManager (suite) <<<
> [junit4] > Throwable #1: java.lang.AssertionError: The test or suite
> printed 16173 bytes to stdout and stderr, even though the limit was set to
> 8192 bytes. Increase the limit with @Limit, ignore it completely with
> @SuppressSysoutChecks or run with -Dtests.verbose=true
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([CBAB78174F1B51A9]:0)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] Completed [546/552 (1!)] on J0 in 355.57s, 11 tests, 1
> failure, 1 error <<< FAILURES!
>
> [...truncated 12649 lines...]
> [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBootstrapTest
> [junit4] 2> 5121336 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 Created dataDir:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/data-dir-186-001
> [junit4] 2> 5121337 WARN
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11
> [junit4] 2> 5121337 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
> w/NUMERIC_DOCVALUES_SYSPROP=false
> [junit4] 2> 5121337 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via:
> @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0,
> clientAuth=0.0/0.0)
> [junit4] 2> 5121338 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4 SecureRandom sanity checks:
> test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
> [junit4] 2> 5121341 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithMultipleReplicas
> [junit4] 2> 5121341 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001
> [junit4] 2> 5121341 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ]
> o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
> [junit4] 2> 5121342 INFO (ZkTestServer Run Thread) [ ]
> o.a.s.c.ZkTestServer Starting server
> [junit4] 2> 5121442 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer start zk server on port:36855
> [junit4] 2> 5121442 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:36855
> [junit4] 2> 5121442 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36855
> [junit4] 2> 5121442 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36855
> [junit4] 2> 5121443 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5121452 INFO
> (zkConnectionManagerCallback-12879-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121452 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121454 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5121455 INFO
> (zkConnectionManagerCallback-12881-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121455 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121456 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5121457 INFO
> (zkConnectionManagerCallback-12883-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121457 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121562 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5121562 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.session node0 Scavenging every 600000ms
> [junit4] 2> 5121564 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@867eda0
> {/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.session node0 Scavenging every 660000ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.session DefaultSessionIdManager workerName=node0
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.session No SessionScavenger set, using defaults
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.session node0 Scavenging every 660000ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d1dd263
> {/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ecf5186
> {/solr,null,AVAILABLE}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.AbstractConnector Started ServerConnector@3473a5ca{ssl,[ssl,
> alpn, http/1.1, h2]}{127.0.0.1:41191}
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
> o.e.j.s.Server Started @5121593ms
> [junit4] 2> 5121565 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
> hostPort=41191}
> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-2) [ ]
> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
> be missing or incomplete.
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter Using logger factory
> org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.AbstractConnector Started ServerConnector@77d77ae2{ssl,[ssl,
> alpn, http/1.1, h2]}{127.0.0.1:37757}
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.e.j.s.Server Started @5121593ms
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
> hostPort=37757}
> [junit4] 2> 5121566 ERROR (jetty-launcher-12884-thread-1) [ ]
> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
> be missing or incomplete.
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter Using logger factory
> org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
> version 9.0.0
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
> version 9.0.0
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
> null
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
> null
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
> 2020-02-06T05:48:19.821958Z
> [junit4] 2> 5121566 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
> 2020-02-06T05:48:19.821957Z
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.AbstractConnector Started ServerConnector@16db638b{ssl,[ssl,
> alpn, http/1.1, h2]}{127.0.0.1:43461}
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.e.j.s.Server Started @5121594ms
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr,
> hostPort=43461}
> [junit4] 2> 5121567 ERROR (jetty-launcher-12884-thread-3) [ ]
> o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may
> be missing or incomplete.
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter Using logger factory
> org.apache.logging.slf4j.Log4jLoggerFactory
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™
> version 9.0.0
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port
> null
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
> [junit4] 2> 5121567 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time:
> 2020-02-06T05:48:19.822974Z
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121568 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5121569 INFO
> (zkConnectionManagerCallback-12886-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121569 INFO
> (zkConnectionManagerCallback-12888-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121569 INFO
> (zkConnectionManagerCallback-12890-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5121569 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5121570 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
> [junit4] 2> 5122191 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
> WhitelistHostChecker [whitelistHosts=null,
> whitelistHostCheckingEnabled=true]
> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@9c25133
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122192 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@9c25133[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@3dc1efed
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122222 WARN (jetty-launcher-12884-thread-1) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@3dc1efed[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122224 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5122229 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5122230 INFO
> (zkConnectionManagerCallback-12899-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122230 INFO (jetty-launcher-12884-thread-1) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5122332 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Waiting for
> client to connect to ZooKeeper
> [junit4] 2> 5122333 INFO
> (zkConnectionManagerCallback-12901-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122333 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.c.ConnectionManager Client is
> connected to ZooKeeper
> [junit4] 2> 5122450 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.OverseerElectionContext I am going
> to be the leader 127.0.0.1:37757_solr
> [junit4] 2> 5122451 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Overseer
> (id=73197244948086791-127.0.0.1:37757_solr-n_0000000000) starting
> [junit4] 2> 5122454 INFO
> (OverseerStateUpdate-73197244948086791-127.0.0.1:37757_solr-n_0000000000)
> [n:127.0.0.1:37757_solr ] o.a.s.c.Overseer Starting to work on the
> main queue : 127.0.0.1:37757_solr
> [junit4] 2> 5122464 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.ZkController Register node as live
> in ZooKeeper:/live_nodes/127.0.0.1:37757_solr
> [junit4] 2> 5122467 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.p.PackageLoader /packages.json
> updated to version -1
> [junit4] 2> 5122467 WARN (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.CoreContainer Not all security
> plugins configured! authentication=disabled authorization=disabled. Solr
> is only as secure as you make it. Consider configuring
> authentication/authorization before exposing Solr to users internal or
> external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5122475 INFO (zkCallback-12900-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
> [junit4] 2> 5122500 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
> collection, keeping metrics history in memory.
> [junit4] 2> 5122523 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_37757.solr.node' (registry 'solr.node') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_37757.solr.jvm' (registry 'solr.jvm') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122532 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_37757.solr.jetty' (registry 'solr.jetty') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122533 INFO (jetty-launcher-12884-thread-1)
> [n:127.0.0.1:37757_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
> definitions underneath
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node1/.
> [junit4] 2> 5122737 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
> WhitelistHostChecker [whitelistHosts=null,
> whitelistHostCheckingEnabled=true]
> [junit4] 2> 5122738 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@2d6e218d
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122739 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@2d6e218d[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@1ed7f896
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122741 WARN (jetty-launcher-12884-thread-3) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@1ed7f896[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5122743 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5122744 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5122746 INFO
> (zkConnectionManagerCallback-12911-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122746 INFO (jetty-launcher-12884-thread-3) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5122848 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Waiting for
> client to connect to ZooKeeper
> [junit4] 2> 5122852 INFO
> (zkConnectionManagerCallback-12913-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5122852 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ConnectionManager Client is
> connected to ZooKeeper
> [junit4] 2> 5122864 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.c.ZkStateReader Updated live nodes
> from ZooKeeper... (0) -> (1)
> [junit4] 2> 5122868 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Publish
> node=127.0.0.1:43461_solr as DOWN
> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.TransientSolrCoreCacheDefault
> Allocating transient cache for 2147483647 transient cores
> [junit4] 2> 5122869 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.ZkController Register node as live
> in ZooKeeper:/live_nodes/127.0.0.1:43461_solr
> [junit4] 2> 5122870 INFO (zkCallback-12900-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
> [junit4] 2> 5122870 INFO (zkCallback-12912-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
> [junit4] 2> 5122871 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.p.PackageLoader /packages.json
> updated to version -1
> [junit4] 2> 5122871 WARN (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.CoreContainer Not all security
> plugins configured! authentication=disabled authorization=disabled. Solr
> is only as secure as you make it. Consider configuring
> authentication/authorization before exposing Solr to users internal or
> external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5122894 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
> collection, keeping metrics history in memory.
> [junit4] 2> 5122918 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_43461.solr.node' (registry 'solr.node') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122929 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_43461.solr.jvm' (registry 'solr.jvm') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122930 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_43461.solr.jetty' (registry 'solr.jetty') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5122931 INFO (jetty-launcher-12884-thread-3)
> [n:127.0.0.1:43461_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
> definitions underneath
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node3/.
> [junit4] 2> 5123682 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized:
> WhitelistHostChecker [whitelistHosts=null,
> whitelistHostCheckingEnabled=true]
> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@32f87da4
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123683 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@32f87da4[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.u.s.S.config Trusting all certificates configured for Client@32434175
> [provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123685 WARN (jetty-launcher-12884-thread-2) [ ]
> o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for
> Client@32434175[provider=null,keyStore=null,trustStore=null]
> [junit4] 2> 5123687 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36855/solr
> [junit4] 2> 5123688 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
> [junit4] 2> 5123693 INFO
> (zkConnectionManagerCallback-12923-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123693 INFO (jetty-launcher-12884-thread-2) [ ]
> o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5123802 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Waiting for
> client to connect to ZooKeeper
> [junit4] 2> 5123803 INFO
> (zkConnectionManagerCallback-12925-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123803 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ConnectionManager Client is
> connected to ZooKeeper
> [junit4] 2> 5123810 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.c.ZkStateReader Updated live nodes
> from ZooKeeper... (0) -> (2)
> [junit4] 2> 5123813 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Publish
> node=127.0.0.1:41191_solr as DOWN
> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.TransientSolrCoreCacheDefault
> Allocating transient cache for 2147483647 transient cores
> [junit4] 2> 5123814 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.ZkController Register node as live
> in ZooKeeper:/live_nodes/127.0.0.1:41191_solr
> [junit4] 2> 5123815 INFO (zkCallback-12912-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123815 INFO (zkCallback-12900-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123817 INFO (zkCallback-12924-thread-1) [ ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
> [junit4] 2> 5123817 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.p.PackageLoader /packages.json
> updated to version -1
> [junit4] 2> 5123818 WARN (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.CoreContainer Not all security
> plugins configured! authentication=disabled authorization=disabled. Solr
> is only as secure as you make it. Consider configuring
> authentication/authorization before exposing Solr to users internal or
> external. See https://s.apache.org/solrsecurity for more info
> [junit4] 2> 5123839 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.h.a.MetricsHistoryHandler No .system
> collection, keeping metrics history in memory.
> [junit4] 2> 5123860 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_41191.solr.node' (registry 'solr.node') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_41191.solr.jvm' (registry 'solr.jvm') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123868 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring
> for 'solr_41191.solr.jetty' (registry 'solr.jetty') enabled at server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@1807b4c8
> [junit4] 2> 5123869 INFO (jetty-launcher-12884-thread-2)
> [n:127.0.0.1:41191_solr ] o.a.s.c.CorePropertiesLocator Found 0 core
> definitions underneath
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-target-001/node2/.
> [junit4] 2> 5123900 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=3
> [junit4] 2> 5123901 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5123902 INFO
> (zkConnectionManagerCallback-12934-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5123903 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5123904 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) ->
> (3)
> [junit4] 2> 5123905 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at
> 127.0.0.1:36855/solr ready
> [junit4] 1> Target zkHost = 127.0.0.1:36855/solr
> [junit4] 2> 5123905 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBootstrapTest_AE624A5368110313-001/cdcr-source-001
> [junit4] 2> 5123906 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ]
> o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
> [junit4] 2> 5123906 INFO (ZkTestServer Run Thread) [ ]
> o.a.s.c.ZkTestServer Starting server
> [junit4] 2> 5124006 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer start zk server on port:40811
> [junit4] 2> 5124006 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:40811
> [junit4] 2> 5124006 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40811
> [junit4] 2> 5124006 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40811
> [junit4] 2> 5124009 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5124011 INFO
> (zkConnectionManagerCallback-12936-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124011 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124013 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5124014 INFO
> (zkConnectionManagerCallback-12938-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124014 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124015 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to
> ZooKeeper
> [junit4] 2> 5124015 INFO
> (zkConnectionManagerCallback-12940-thread-1) [ ]
> o.a.s.c.c.ConnectionManager zkClient has connected
> [junit4] 2> 5124015 INFO
> (TEST-CdcrBootstrapTest.testBootstrapWithMultipleReplicas-seed#[AE624A5368110313])
> [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-3) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-2) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5124123 WARN (jetty-launcher-12941-thread-1) [ ]
> o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content,
> break even is size 23
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-3) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
> o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-1) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
> [junit4] 2> 5124123 INFO (jetty-launcher-12941-thread-2) [ ]
> o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z;
> git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 11.0.4+10-LTS
>
>
> [...truncated too long message...]
>
> p(Native Method)
> [junit4] 2>
> app//com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
> [junit4] 2>
> [junit4] 2> 5279049 INFO
> (SUITE-CdcrBootstrapTest-seed#[AE624A5368110313]-worker) [ ]
> o.a.s.SolrTestCaseJ4
> ------------------------------------------------------- Done waiting for
> tracked resources to be released
> [junit4] 2> NOTE: test params are: codec=SimpleText,
> sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@8a39653),
> locale=en-SD, timezone=VST
> [junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation
> 11.0.4 (64-bit)/cpus=4,threads=1,free=386818192,total=536870912
> [junit4] 2> NOTE: All tests run in this JVM: [AuditLoggerPluginTest,
> CdcrUpdateLogTest, SolrCloudExampleTest, ZookeeperStatusHandlerTest,
> UUIDFieldTest, NodeLostTriggerTest,
> TestOverriddenPrefixQueryForCustomFieldType, TestRandomFaceting,
> TestIBSimilarityFactory, TestUtils, UpdateLogTest, TestConfigOverlay,
> PeerSyncWithBufferUpdatesTest, TestHashQParserPlugin,
> TestLegacyBM25SimilarityFactory, CachingDirectoryFactoryTest,
> SortByFunctionTest, TestZkChroot, IgnoreLargeDocumentProcessorFactoryTest,
> TestPHPSerializedResponseWriter, TestInPlaceUpdatesRequiredField,
> TestJettySolrRunner, TestSerializedLuceneMatchVersion, DOMUtilTest,
> CoreAdminHandlerTest, TestManagedResource, TestSolrIndexConfig,
> BJQParserTest, TestSimLargeCluster, TestStressThreadBackup,
> AsyncCallRequestStatusResponseTest, TestImplicitCoreProperties,
> DistributedFacetPivotLongTailTest, WrapperMergePolicyFactoryTest,
> TestConfigSetsAPIExclusivity, TestRTimerTree, SolrJmxReporterCloudTest,
> QueryElevationComponentTest, NotRequiredUniqueKeyTest,
> ChaosMonkeyNothingIsSafeTest, MBeansHandlerTest,
> TestHighlightDedupGrouping, ZkShardTermsTest, TestJmxIntegration,
> CurrencyRangeFacetCloudTest, ShowFileRequestHandlerTest, TestRTGBase,
> TestCloudSearcherWarming, TestRawTransformer,
> SignificantTermsQParserPluginTest, MinimalSchemaTest,
> TestGeoJSONResponseWriter, AutoAddReplicasIntegrationTest,
> TestChildDocTransformer, TestStandardQParsers, TestNestedDocsSort,
> TestReRankQParserPlugin, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest,
> SolrCoreCheckLockOnStartupTest, TestSSLRandomization,
> TestLegacyNumericRangeQueryBuilder, HdfsBasicDistributedZkTest,
> OverseerSolrResponseTest, TestPKIAuthenticationPlugin,
> DistributedFacetPivotWhiteBoxTest, TermVectorComponentDistributedTest,
> Tagger2Test, PeerSyncWithLeaderTest, TestLRUStatsCacheCloud,
> ZkStateReaderTest, LeaderFailoverAfterPartitionTest,
> MultiSolrCloudTestCaseTest, LeaderElectionContextKeyTest, TestTlogReplica,
> SearchRateTriggerIntegrationTest, TestCollationField,
> TestHighFrequencyDictionaryFactory, NodeAddedTriggerIntegrationTest,
> TestConfigReload, TestSimPolicyCloud,
> DistributedQueryComponentOptimizationTest, SolrCmdDistributorTest,
> TestDynamicLoading, V2StandaloneTest, TestFiltering, CollectionPropsTest,
> TestSimComputePlanAction, CloudExitableDirectoryReaderTest, PrimUtilsTest,
> HttpPartitionOnCommitTest, ForceLeaderWithTlogReplicasTest,
> TestSchemaManager, TestMultiValuedNumericRangeQuery,
> TestClassicSimilarityFactory, TestBinaryResponseWriter,
> DefaultValueUpdateProcessorTest, TestSolrCloudWithKerberosAlt,
> CopyFieldTest, HdfsNNFailoverTest, TestManagedSynonymFilterFactory,
> HdfsTlogReplayBufferedWhileIndexingTest, AdminHandlersProxyTest,
> TestSolrTestCaseJ4, TestSolrConfigHandler, QueryParsingTest,
> TestCloudJSONFacetJoinDomain, TestCollectionAPIs, TestSolrJ,
> TestStressInPlaceUpdates, HttpTriggerListenerTest,
> SpellCheckCollatorWithCollapseTest,
> OverseerCollectionConfigSetProcessorTest,
> DistributedFacetSimpleRefinementLongTailTest,
> ScheduledMaintenanceTriggerTest, DistributedSuggestComponentTest,
> TestPerFieldSimilarityWithDefaultOverride, TestFieldTypeCollectionResource,
> TestExtendedDismaxParser, TestDelegationWithHadoopAuth,
> CloudMLTQParserTest, MetricsHistoryWithAuthIntegrationTest,
> ClassificationUpdateProcessorIntegrationTest,
> TestLocalFSCloudBackupRestore, TestSizeLimitedDistributedMap,
> XMLAtomicUpdateMultivalueTest, ShardSplitTest, MoveReplicaHDFSFailoverTest,
> TestSurroundQueryParser, CoreMergeIndexesAdminHandlerTest,
> BasicAuthStandaloneTest, TestSolrCachePerf, TestDistributedSearch,
> StressHdfsTest, TestBulkSchemaConcurrent, TestDownShardTolerantSearch,
> HdfsRestartWhileUpdatingTest, TestBadConfig, SolrInfoBeanTest,
> XmlInterpolationTest, TestDistributedStatsComponentCardinality,
> LeaderVoteWaitTimeoutTest, SearchHandlerTest, TestMacros,
> BasicDistributedZk2Test, CollectionsAPIAsyncDistributedZkTest,
> TestExactStatsCacheCloud, TestLeaderElectionZkExpiry,
> TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplica,
> TestPullReplicaErrorHandling, TestRandomRequestDistribution,
> TestRebalanceLeaders, TestTolerantUpdateProcessorRandomCloud,
> TestUtilizeNode, TestWaitForStateWithJettyShutdowns,
> TriLevelCompositeIdRoutingTest, UnloadDistributedZkTest,
> VMParamsZkACLAndCredentialsProvidersTest, ZkCLITest,
> ConcurrentDeleteAndCreateCollectionTest,
> TestCollectionsAPIViaSolrCloudCluster, AutoScalingHandlerTest,
> ExecutePlanActionTest, IndexSizeTriggerSizeEstimationTest,
> MetricTriggerIntegrationTest, SystemLogListenerTest, TestPolicyCloud,
> TriggerCooldownIntegrationTest, TriggerSetPropertiesIntegrationTest,
> TestSimDistributedQueue, TestSimExecutePlanAction, TestSimExtremeIndexing,
> TestSimTriggerIntegration, TestSimUtils, TestSnapshotCloudManager,
> BaseCdcrDistributedZkTest, CdcrBidirectionalTest, CdcrBootstrapTest]
> [junit4] 2> NOTE: download the large Jenkins line-docs file by
> running 'ant get-jenkins-line-docs' in the lucene directory.
> [junit4] 2> NOTE: reproduce with: ant test
> -Dtestcase=CdcrBootstrapTest -Dtests.seed=AE624A5368110313
> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
> -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
> -Dtests.locale=en-SD -Dtests.timezone=VST -Dtests.asserts=true
> -Dtests.file.encoding=UTF-8
> [junit4] ERROR 0.00s J1 | CdcrBootstrapTest (suite) <<<
> [junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker
> found 1 object(s) that were not released!!! [RawDirectoryWrapper]
> [junit4] >
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper
> [junit4] > at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> [junit4] > at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> [junit4] > at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> [junit4] > at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> [junit4] > at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> [junit4] > at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> [junit4] > at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> [junit4] > at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> [junit4] > at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> [junit4] > at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> [junit4] > at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [junit4] > at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] > expected null, but was:<ObjectTracker found 1 object(s)
> that were not released!!! [RawDirectoryWrapper]
> [junit4] >
> org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
> org.apache.lucene.store.RawDirectoryWrapper
> [junit4] > at
> org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
> [junit4] > at
> org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
> [junit4] > at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
> [junit4] > at
> org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
> [junit4] > at
> org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
> [junit4] > at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:772)
> [junit4] > at
> org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:727)
> [junit4] > at
> com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:202)
> [junit4] > at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> [junit4] > at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
> [junit4] > at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [junit4] > at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] > >
> [junit4] > at
> __randomizedtesting.SeedInfo.seed([AE624A5368110313]:0)
> [junit4] > at
> org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:345)
> [junit4] > at
> jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
> [junit4] > at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [junit4] > at
> java.base/java.lang.reflect.Method.invoke(Method.java:566)
> [junit4] > at java.base/java.lang.Thread.run(Thread.java:834)
> [junit4] Completed [572/900 (1!)] on J1 in 157.86s, 4 tests, 1 failure,
> 1 skipped <<< FAILURES!
>
> [...truncated 53511 lines...]
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: builds-unsubscribe@lucene.apache.org
> For additional commands, e-mail: builds-help@lucene.apache.org
--
Adrien