svn commit: r1553644 - in /lucene/dev/trunk/solr: CHANGES.txt core/src/java/org/apache/solr/update/SolrIndexSplitter.java

[sh...@apache.org]

Author: shalin
Date: Fri Dec 27 08:36:23 2013
New Revision: 1553644

URL: http://svn.apache.org/r1553644
Log:
SOLR-5214: Reduce memory usage for shard splitting by merging segments one at a time

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1553644&r1=1553643&r2=1553644&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Fri Dec 27 08:36:23 2013
@@ -236,6 +236,9 @@ Optimizations
 * SOLR-2960: fix DIH XPathEntityProcessor to add the correct number of "null"
   placeholders for multi-valued fields (Michael Watts via James Dyer)
 
+* SOLR-5214: Reduce memory usage for shard splitting by merging segments one
+  at a time. (Christine Poerschke via shalin)
+
 Other Changes
 ---------------------
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1553644&r1=1553643&r2=1553644&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Fri Dec 27 08:36:23 2013
@@ -105,13 +105,8 @@ public class SolrIndexSplitter {
     // - need to worry about if IW.addIndexes does a sync or not...
     // - would be more efficient on the read side, but prob less efficient merging
 
-    IndexReader[] subReaders = new IndexReader[leaves.size()];
     for (int partitionNumber=0; partitionNumber<numPieces; partitionNumber++) {
-      log.info("SolrIndexSplitter: partition #" + partitionNumber + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
-
-      for (int segmentNumber = 0; segmentNumber<subReaders.length; segmentNumber++) {
-        subReaders[segmentNumber] = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
-      }
+      log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
 
       boolean success = false;
 
@@ -130,8 +125,12 @@ public class SolrIndexSplitter {
       }
 
       try {
-        // This merges the subreaders and will thus remove deletions (i.e. no optimize needed)
-        iw.addIndexes(subReaders);
+        // This removes deletions but optimize might still be needed because sub-shards will have the same number of segments as the parent shard.
+        for (int segmentNumber = 0; segmentNumber<leaves.size(); segmentNumber++) {
+          log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : "") + " segment #"+segmentNumber + " segmentCount=" + leaves.size());
+          IndexReader subReader = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
+          iw.addIndexes(subReader);
+        }
         success = true;
       } finally {
         if (iwRef != null) {