You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2020/08/07 17:54:51 UTC
[lucene-solr] branch master updated: SOLR-14582: Expose
IWC.setMaxCommitMergeWaitMillis in Solr's index config (#1602)
This is an automated email from the ASF dual-hosted git repository.
tflobbe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new e6275d9 SOLR-14582: Expose IWC.setMaxCommitMergeWaitMillis in Solr's index config (#1602)
e6275d9 is described below
commit e6275d9970702c250b69792b5491ef9d94b08638
Author: Tomas Fernandez Lobbe <tf...@apache.org>
AuthorDate: Fri Aug 7 10:54:36 2020 -0700
SOLR-14582: Expose IWC.setMaxCommitMergeWaitMillis in Solr's index config (#1602)
---
solr/CHANGES.txt | 3 +++
.../org/apache/solr/update/SolrIndexConfig.java | 22 +++++++++++++++++++++-
.../conf/solrconfig.snippet.randomindexconfig.xml | 1 +
.../apache/solr/update/SolrIndexConfigTest.java | 19 +++++++++++++++++++
4 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c6e6f13..2cc9710 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -126,6 +126,9 @@ New Features
* SOLR-14604: Add the ability to uninstall a package from with the Package CLI. (MarcusSorealheis)
+* SOLR-14582: Expose IWC.setMaxCommitMergeWaitMillis in Solr's index config. This is an expert config option that can be
+ set when using a custom MergePolicy (doesn't have any effect on the default MP) (Tomás Fernández Löbbe)
+
Improvements
---------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
index e189ad1..50bda0e 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
@@ -68,6 +68,19 @@ public class SolrIndexConfig implements MapSerializable {
public final double ramBufferSizeMB;
public final int ramPerThreadHardLimitMB;
+ /**
+ * <p>
+ * When using a custom merge policy that allows triggering synchronous merges on commit
+ * (see {@link MergePolicy#findFullFlushMerges(org.apache.lucene.index.MergeTrigger, org.apache.lucene.index.SegmentInfos, org.apache.lucene.index.MergePolicy.MergeContext)}),
+ * a timeout (in milliseconds) can be set for those merges to finish. Use {@code <maxCommitMergeWaitTime>1000</maxCommitMergeWaitTime>} in the {@code <indexConfig>} section.
+ * See {@link IndexWriterConfig#setMaxCommitMergeWaitMillis(long)}.
+ * </p>
+ * <p>
+ * Note that as of Solr 8.6, no {@code MergePolicy} shipped with Lucene/Solr make use of
+ * {@code MergePolicy.findFullFlushMerges}, which means this setting has no effect unless a custom {@code MergePolicy} is used.
+ * </p>
+ */
+ public final int maxCommitMergeWaitMillis;
public final int writeLockTimeout;
public final String lockType;
@@ -87,6 +100,7 @@ public class SolrIndexConfig implements MapSerializable {
maxBufferedDocs = -1;
ramBufferSizeMB = 100;
ramPerThreadHardLimitMB = -1;
+ maxCommitMergeWaitMillis = -1;
writeLockTimeout = -1;
lockType = DirectoryFactory.LOCK_TYPE_NATIVE;
mergePolicyFactoryInfo = null;
@@ -129,8 +143,9 @@ public class SolrIndexConfig implements MapSerializable {
true);
useCompoundFile = solrConfig.getBool(prefix+"/useCompoundFile", def.useCompoundFile);
- maxBufferedDocs=solrConfig.getInt(prefix+"/maxBufferedDocs",def.maxBufferedDocs);
+ maxBufferedDocs = solrConfig.getInt(prefix+"/maxBufferedDocs", def.maxBufferedDocs);
ramBufferSizeMB = solrConfig.getDouble(prefix+"/ramBufferSizeMB", def.ramBufferSizeMB);
+ maxCommitMergeWaitMillis = solrConfig.getInt(prefix+"/maxCommitMergeWaitTime", def.maxCommitMergeWaitMillis);
// how do we validate the value??
ramPerThreadHardLimitMB = solrConfig.getInt(prefix+"/ramPerThreadHardLimitMB", def.ramPerThreadHardLimitMB);
@@ -185,6 +200,7 @@ public class SolrIndexConfig implements MapSerializable {
"maxBufferedDocs", maxBufferedDocs,
"ramBufferSizeMB", ramBufferSizeMB,
"ramPerThreadHardLimitMB", ramPerThreadHardLimitMB,
+ "maxCommitMergeWaitTime", maxCommitMergeWaitMillis,
"writeLockTimeout", writeLockTimeout,
"lockType", lockType,
"infoStreamEnabled", infoStream != InfoStream.NO_OUTPUT);
@@ -230,6 +246,10 @@ public class SolrIndexConfig implements MapSerializable {
if (ramPerThreadHardLimitMB != -1) {
iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB);
}
+
+ if (maxCommitMergeWaitMillis > 0) {
+ iwc.setMaxCommitMergeWaitMillis(maxCommitMergeWaitMillis);
+ }
iwc.setSimilarity(schema.getSimilarity());
MergePolicy mergePolicy = buildMergePolicy(core.getResourceLoader(), schema);
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
index 20ddf96..de5c714 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
@@ -32,6 +32,7 @@ A solrconfig.xml snippet containing indexConfig settings for randomized testing.
<maxBufferedDocs>${solr.tests.maxBufferedDocs}</maxBufferedDocs>
<ramBufferSizeMB>${solr.tests.ramBufferSizeMB}</ramBufferSizeMB>
+ <maxCommitMergeWaitTime>${solr.tests.maxCommitMergeWaitTime:-1}</maxCommitMergeWaitTime>
<ramPerThreadHardLimitMB>${solr.tests.ramPerThreadHardLimitMB}</ramPerThreadHardLimitMB>
<mergeScheduler class="${solr.tests.mergeScheduler}" />
diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
index 5ae02af..2e4e597 100644
--- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
@@ -36,6 +36,7 @@ import org.apache.solr.core.TestMergePolicyConfig;
import org.apache.solr.index.SortingMergePolicy;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
+import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -58,6 +59,12 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 {
initCore(solrConfigFileName,schemaFileName);
}
+ @After
+ public void tearDown() throws Exception {
+ System.clearProperty("solr.tests.maxCommitMergeWait");
+ super.tearDown();
+ }
+
private final Path instanceDir = TEST_PATH().resolve("collection1");
@Test
@@ -177,6 +184,8 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 {
++mSizeExpected; assertTrue(m.get("maxBufferedDocs") instanceof Integer);
++mSizeExpected; assertTrue(m.get("ramBufferSizeMB") instanceof Double);
+
+ ++mSizeExpected; assertTrue(m.get("maxCommitMergeWaitTime") instanceof Integer);
++mSizeExpected; assertTrue(m.get("ramPerThreadHardLimitMB") instanceof Integer);
@@ -208,4 +217,14 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 {
assertEquals(mSizeExpected, m.size());
}
+
+ public void testMaxCommitMergeWaitTime() throws Exception {
+ SolrConfig sc = new SolrConfig(TEST_PATH().resolve("collection1"), "solrconfig-test-misc.xml");
+ assertEquals(-1, sc.indexConfig.maxCommitMergeWaitMillis);
+ assertEquals(IndexWriterConfig.DEFAULT_MAX_COMMIT_MERGE_WAIT_MILLIS, sc.indexConfig.toIndexWriterConfig(h.getCore()).getMaxCommitMergeWaitMillis());
+ System.setProperty("solr.tests.maxCommitMergeWaitTime", "10");
+ sc = new SolrConfig(TEST_PATH().resolve("collection1"), "solrconfig-test-misc.xml");
+ assertEquals(10, sc.indexConfig.maxCommitMergeWaitMillis);
+ assertEquals(10, sc.indexConfig.toIndexWriterConfig(h.getCore()).getMaxCommitMergeWaitMillis());
+ }
}