You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/04/23 17:35:26 UTC
[22/40] lucene-solr:jira/solr-11833: SOLR-11200: A new CMS config
option 'ioThrottle' to manually enable/disable
ConcurrentMergeSchedule.doAutoIOThrottle. (Amrit Sarkar,
Nawab Zada Asad iqbal)
SOLR-11200: A new CMS config option 'ioThrottle' to manually enable/disable ConcurrentMergeSchedule.doAutoIOThrottle. (Amrit Sarkar, Nawab Zada Asad iqbal)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4eead83a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4eead83a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4eead83a
Branch: refs/heads/jira/solr-11833
Commit: 4eead83a83235b235145f07f0a625055b860ad65
Parents: cf05e17
Author: Dawid Weiss <dw...@apache.org>
Authored: Fri Apr 20 11:34:04 2018 +0200
Committer: Dawid Weiss <dw...@apache.org>
Committed: Fri Apr 20 11:34:04 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../org/apache/solr/update/SolrIndexConfig.java | 4 +++
.../solrconfig-concurrentmergescheduler.xml | 37 ++++++++++++++++++++
.../apache/solr/update/SolrIndexConfigTest.java | 24 +++++++++++++
4 files changed, 68 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4eead83a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index be3f704..516a0d7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -69,6 +69,9 @@ Upgrade Notes
New Features
----------------------
+* SOLR-11200: A new CMS config option 'ioThrottle' to manually enable/disable
+ ConcurrentMergeSchedule.doAutoIOThrottle. (Amrit Sarkar, Nawab Zada Asad iqbal via Dawid Weiss)
+
* SOLR-11670: Implement a periodic house-keeping task. This uses a scheduled autoscaling trigger and
currently performs cleanup of old inactive shards. (ab, shalin)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4eead83a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
index c663783..48b2417 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
@@ -299,6 +299,10 @@ public class SolrIndexConfig implements MapSerializable {
maxThreadCount = ((ConcurrentMergeScheduler) scheduler).getMaxThreadCount();
}
((ConcurrentMergeScheduler)scheduler).setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
+ Boolean ioThrottle = (Boolean) args.remove("ioThrottle");
+ if (ioThrottle != null && !ioThrottle) { //by-default 'enabled'
+ ((ConcurrentMergeScheduler) scheduler).disableAutoIOThrottle();
+ }
SolrPluginUtils.invokeSetters(scheduler, args);
} else {
SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4eead83a/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml
new file mode 100644
index 0000000..140c4cf
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <indexConfig>
+ <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+ <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory" />
+ <mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler">
+ <int name="maxMergeCount">987</int>
+ <int name="maxThreadCount">42</int>
+ <bool name="ioThrottle">false</bool>
+ </mergeScheduler>
+ </indexConfig>
+
+ <requestHandler name="/select" class="solr.SearchHandler"></requestHandler>
+
+</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4eead83a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
index ec5719c..d5ebf05 100644
--- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
@@ -48,6 +48,7 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 {
private static final String solrConfigFileName = "solrconfig.xml";
private static final String solrConfigFileNameWarmerRandomMergePolicyFactory = "solrconfig-warmer-randommergepolicyfactory.xml";
private static final String solrConfigFileNameTieredMergePolicyFactory = "solrconfig-tieredmergepolicyfactory.xml";
+ private static final String solrConfigFileNameConnMSPolicyFactory = "solrconfig-concurrentmergescheduler.xml";
private static final String solrConfigFileNameSortingMergePolicyFactory = "solrconfig-sortingmergepolicyfactory.xml";
private static final String schemaFileName = "schema.xml";
@@ -93,6 +94,29 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 {
ConcurrentMergeScheduler ms = (ConcurrentMergeScheduler) iwc.getMergeScheduler();
assertEquals("ms.maxMergeCount", 987, ms.getMaxMergeCount());
assertEquals("ms.maxThreadCount", 42, ms.getMaxThreadCount());
+ assertEquals("ms.isAutoIOThrottle", true, ms.getAutoIOThrottle());
+
+ }
+
+ @Test
+ public void testConcurrentMergeSchedularSolrIndexConfigCreation() throws Exception {
+ String solrConfigFileName = solrConfigFileNameConnMSPolicyFactory;
+ SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileName, null);
+ SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null);
+ IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig);
+
+ h.getCore().setLatestSchema(indexSchema);
+ IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore());
+
+ assertNotNull("null mp", iwc.getMergePolicy());
+ assertTrue("mp is not TieredMergePolicy", iwc.getMergePolicy() instanceof TieredMergePolicy);
+
+ assertNotNull("null ms", iwc.getMergeScheduler());
+ assertTrue("ms is not CMS", iwc.getMergeScheduler() instanceof ConcurrentMergeScheduler);
+ ConcurrentMergeScheduler ms = (ConcurrentMergeScheduler) iwc.getMergeScheduler();
+ assertEquals("ms.maxMergeCount", 987, ms.getMaxMergeCount());
+ assertEquals("ms.maxThreadCount", 42, ms.getMaxThreadCount());
+ assertEquals("ms.isAutoIOThrottle", false, ms.getAutoIOThrottle());
}