You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2020/04/24 06:39:49 UTC
[lucene-solr] branch branch_8x updated: Ensure we use a sane IWC
for tests adding many documents.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new a5cda32 Ensure we use a sane IWC for tests adding many documents.
a5cda32 is described below
commit a5cda3294962617b57d0822cccb8a150c33cc58d
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Fri Apr 24 08:36:06 2020 +0200
Ensure we use a sane IWC for tests adding many documents.
This test produced tons of files on nighly builds causing
TooManyOpenFilesExceptions likely due to not using CFS on flush
and/or very small maxMergeSize values.
---
.../apache/lucene/search/TestSearcherManager.java | 4 +--
.../ThreadedIndexingAndSearchingTestCase.java | 41 ++++++++++++----------
2 files changed, 25 insertions(+), 20 deletions(-)
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java b/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
index 3d3ec96..9d5e087 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
@@ -556,7 +556,7 @@ public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
AtomicReference<IndexWriter> writerRef = new AtomicReference<>();
final MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(IndexWriter.MAX_TERM_LENGTH);
- writerRef.set(new IndexWriter(dir, newIndexWriterConfig(analyzer)));
+ writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNigtly(newIndexWriterConfig(analyzer))));
AtomicReference<SearcherManager> mgrRef = new AtomicReference<>();
mgrRef.set(new SearcherManager(writerRef.get(), null));
@@ -580,7 +580,7 @@ public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
} else {
w.rollback();
}
- writerRef.set(new IndexWriter(dir, newIndexWriterConfig(analyzer)));
+ writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNigtly(newIndexWriterConfig(analyzer))));
}
}
if (VERBOSE) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
index 079f215..5691979 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
@@ -424,6 +424,28 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
private final Map<SegmentCoreReaders,Boolean> warmed = Collections.synchronizedMap(new WeakHashMap<SegmentCoreReaders,Boolean>());
+ protected static IndexWriterConfig ensureSaneIWCOnNigtly(IndexWriterConfig conf) {
+ if (LuceneTestCase.TEST_NIGHTLY) {
+ // newIWConfig makes smallish max seg size, which
+ // results in tons and tons of segments for this test
+ // when run nightly:
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof TieredMergePolicy) {
+ ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.);
+ } else if (mp instanceof LogByteSizeMergePolicy) {
+ ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.);
+ } else if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setMaxMergeDocs(100000);
+ }
+ // when running nightly, merging can still have crazy parameters,
+ // and might use many per-field codecs. turn on CFS for IW flushes
+ // and ensure CFS ratio is reasonable to keep it contained.
+ conf.setUseCompoundFile(true);
+ mp.setNoCFSRatio(Math.max(0.25d, mp.getNoCFSRatio()));
+ }
+ return conf;
+ }
+
public void runTest(String testName) throws Exception {
failed.set(false);
@@ -448,24 +470,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
((MockRandomMergePolicy)conf.getMergePolicy()).setDoNonBulkMerges(false);
}
- if (LuceneTestCase.TEST_NIGHTLY) {
- // newIWConfig makes smallish max seg size, which
- // results in tons and tons of segments for this test
- // when run nightly:
- MergePolicy mp = conf.getMergePolicy();
- if (mp instanceof TieredMergePolicy) {
- ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.);
- } else if (mp instanceof LogByteSizeMergePolicy) {
- ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.);
- } else if (mp instanceof LogMergePolicy) {
- ((LogMergePolicy) mp).setMaxMergeDocs(100000);
- }
- // when running nightly, merging can still have crazy parameters,
- // and might use many per-field codecs. turn on CFS for IW flushes
- // and ensure CFS ratio is reasonable to keep it contained.
- conf.setUseCompoundFile(true);
- mp.setNoCFSRatio(Math.max(0.25d, mp.getNoCFSRatio()));
- }
+ ensureSaneIWCOnNigtly(conf);
conf.setMergedSegmentWarmer((reader) -> {
if (VERBOSE) {