You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2015/05/20 22:57:30 UTC
svn commit: r1680673 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/update/
solr/core/src/test-files/solr/collection1/conf/
solr/core/src/test/org/apache/solr/core/ solr/server/
solr/server/solr/configsets/d...
Author: tflobbe
Date: Wed May 20 20:57:30 2015
New Revision: 1680673
URL: http://svn.apache.org/r1680673
Log:
SOLR-7463: Stop forcing MergePolicy's ''noCFSRatio'' based on the IWC ''useCompoundFile'' configuration. Use the MP's default if not specifically set
Added:
lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-nocfs.xml
- copied unchanged from r1680665, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-nocfs.xml
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java
lucene/dev/branches/branch_5x/solr/server/ (props changed)
lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1680673&r1=1680672&r2=1680673&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Wed May 20 20:57:30 2015
@@ -57,6 +57,15 @@ Upgrading from Solr 5.1
* The package structure under org.apache.solr.client.solrj.io has been changed to support
the Streaming Expression Language (SOLR-7377). Any code written with the 5.1 Streaming API will have to
be updated to reflect these changes.
+
+* Merge Policy's "noCFSRatio" is no longer set based on <useCompoundFile> element in the indexConfig section
+ of solrconfig.xml. This means that Solr will start using Lucene's default for MP "noCFSRatio", with this
+ new default Solr will decide if a segment should use cfs or not based on the size of the segment in relation
+ the size of the complete index. For TieredMergePolicy for example (current default), segments will use cfs
+ if they are less than 10% of the index, otherwise cfs is disabled. Old values for this setting
+ (1.0 for useCompoundFile=true and 0.0 for useCompoundFile=false) as well as any other value can be set
+ inside the <mergePolicy> element in solrconfig.xml. <useCompoundFile> will only apply to newly created
+ segments. See SOLR-7463.
Detailed Change List
@@ -352,12 +361,15 @@ Other Changes
* SOLR-7541: Removed CollectionsHandler#createNodeIfNotExists. All calls made to this method now call
ZkCmdExecutor#ensureExists as they were doing the same thing. Also ZkCmdExecutor#ensureExists now respects the
CreateMode passed to it. (Varun Thacker)
-
+
* SOLR-6820: Make the number of version buckets used by the UpdateLog configurable as
increasing beyond the default 256 has been shown to help with high volume indexing
performance in SolrCloud; helps overcome a limitation where Lucene uses the request
thread to perform expensive index housekeeping work. (Mark Miller, yonik, Timothy Potter)
+* SOLR-7463: Stop forcing MergePolicy's "NoCFSRatio" based on the IWC "useCompoundFile" configuration
+ (Tomás Fernández Löbbe)
+
================== 5.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1680673&r1=1680672&r2=1680673&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java Wed May 20 20:57:30 2015
@@ -259,18 +259,13 @@ public class SolrIndexConfig implements
if (maxMergeDocs != -1)
logMergePolicy.setMaxMergeDocs(maxMergeDocs);
- logMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0);
-
if (mergeFactor != -1)
logMergePolicy.setMergeFactor(mergeFactor);
-
} else if (policy instanceof TieredMergePolicy) {
TieredMergePolicy tieredMergePolicy = (TieredMergePolicy) policy;
fixUseCFMergePolicyInitArg(TieredMergePolicy.class);
- tieredMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0);
-
if (mergeFactor != -1) {
tieredMergePolicy.setMaxMergeAtOnce(mergeFactor);
tieredMergePolicy.setSegmentsPerTier(mergeFactor);
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java?rev=1680673&r1=1680672&r2=1680673&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java Wed May 20 20:57:30 2015
@@ -17,21 +17,21 @@ package org.apache.solr.core;
* limitations under the License.
*/
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.solr.update.SolrIndexConfigTest;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.TieredMergePolicy;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.LogByteSizeMergePolicy;
import org.apache.lucene.index.LogDocMergePolicy;
-import org.apache.solr.util.RefCounted;
-import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.update.SolrIndexConfigTest;
+import org.apache.solr.util.RefCounted;
import org.junit.After;
-import java.util.concurrent.atomic.AtomicInteger;
/** @see SolrIndexConfigTest */
public class TestMergePolicyConfig extends SolrTestCaseJ4 {
@@ -42,6 +42,22 @@ public class TestMergePolicyConfig exten
public void after() throws Exception {
deleteCore();
}
+
+ public void testSetNoCFSMergePolicyConfig() throws Exception {
+ final boolean useCompoundFile = random().nextBoolean();
+ System.setProperty("testSetNoCFSMergePolicyConfig.useCompoundFile", String.valueOf(useCompoundFile));
+ try {
+ initCore("solrconfig-mergepolicy-nocfs.xml","schema-minimal.xml");
+ IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore());
+ assertEquals(useCompoundFile, iwc.getUseCompoundFile());
+
+ TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class,
+ iwc.getMergePolicy());
+ assertEquals(0.5D, tieredMP.getNoCFSRatio(), 0.0D);
+ } finally {
+ System.getProperties().remove("testSetNoCFSMergePolicyConfig.useCompoundFile");
+ }
+ }
public void testDefaultMergePolicyConfig() throws Exception {
initCore("solrconfig-mergepolicy-defaults.xml","schema-minimal.xml");
@@ -50,7 +66,7 @@ public class TestMergePolicyConfig exten
TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class,
iwc.getMergePolicy());
- assertEquals(0.0D, tieredMP.getNoCFSRatio(), 0.0D);
+ assertEquals(TieredMergePolicy.DEFAULT_NO_CFS_RATIO, tieredMP.getNoCFSRatio(), 0.0D);
assertCommitSomeNewDocs();
assertCompoundSegments(h.getCore(), false);
@@ -70,7 +86,6 @@ public class TestMergePolicyConfig exten
assertEquals(7, tieredMP.getMaxMergeAtOnce());
assertEquals(7.0D, tieredMP.getSegmentsPerTier(), 0.0D);
- assertEquals(expectCFS ? 1.0D : 0.0D, tieredMP.getNoCFSRatio(), 0.0D);
assertCommitSomeNewDocs();
assertCompoundSegments(h.getCore(), expectCFS);
Modified: lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml?rev=1680673&r1=1680672&r2=1680673&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml (original)
+++ lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml Wed May 20 20:57:30 2015
@@ -197,6 +197,7 @@
<mergePolicy class="org.apache.lucene.index.TieredMergePolicy">
<int name="maxMergeAtOnce">10</int>
<int name="segmentsPerTier">10</int>
+ <double name="noCFSRatio">0.1</double>
</mergePolicy>
-->
Modified: lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml?rev=1680673&r1=1680672&r2=1680673&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml (original)
+++ lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml Wed May 20 20:57:30 2015
@@ -199,6 +199,7 @@
<mergePolicy class="org.apache.lucene.index.TieredMergePolicy">
<int name="maxMergeAtOnce">10</int>
<int name="segmentsPerTier">10</int>
+ <double name="noCFSRatio">0.1</double>
</mergePolicy>
-->