svn commit: r1670963 - in /lucene/dev/branches/lucene_solr_5_1/lucene: CHANGES.txt core/src/java/org/apache/lucene/index/IndexWriter.java core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java

[mi...@apache.org]

Author: mikemccand
Date: Thu Apr  2 20:42:04 2015
New Revision: 1670963

URL: http://svn.apache.org/r1670963
Log:
LUCENE-6386: correct javadocs about temp disk space required for forceMerge(1)

Modified:
    lucene/dev/branches/lucene_solr_5_1/lucene/CHANGES.txt
    lucene/dev/branches/lucene_solr_5_1/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/lucene_solr_5_1/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java

Modified: lucene/dev/branches/lucene_solr_5_1/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_1/lucene/CHANGES.txt?rev=1670963&r1=1670962&r2=1670963&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_1/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_5_1/lucene/CHANGES.txt Thu Apr  2 20:42:04 2015
@@ -62,6 +62,10 @@ Bug Fixes
   DocumentsWriterStallControl to prevent hangs during indexing if we
   miss a .notify/All somewhere (Mike McCandless)
 
+* LUCENE-6386: Correct IndexWriter.forceMerge documentation to state
+  that up to 3X (X = current index size) spare disk space may be needed
+  to complete forceMerge(1).  (Robert Muir, Shai Erera, Mike McCandless)
+
 Optimizations
 
 * LUCENE-6183, LUCENE-5647: Avoid recompressing stored fields

Modified: lucene/dev/branches/lucene_solr_5_1/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_1/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1670963&r1=1670962&r2=1670963&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_1/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_5_1/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Thu Apr  2 20:42:04 2015
@@ -1542,14 +1542,15 @@ public class IndexWriter implements Clos
    * longer be changed).</p>
    *
    * <p>Note that this requires free space that is proportional
-   * to the size of the index in your Directory (2X if you're
-   * using compound file format). For example, if your index
-   * size is 10 MB then you need an additional 10 MB free for
-   * this to complete (20 MB if you're using compound file
-   * format). This is also affected by the {@link Codec} that
-   * is used to execute the merge, and may result in even a
-   * bigger index. Also, it's best to call {@link #commit()}
-   * afterwards, to allow IndexWriter to free up disk space.</p>
+   * to the size of the index in your Directory: 2X if you are
+   * not using compound file format, and 3X if you are.
+   * For example, if your index size is 10 MB then you need
+   * an additional 20 MB free for this to complete (30 MB if
+   * you're using compound file format). This is also affected
+   * by the {@link Codec} that is used to execute the merge,
+   * and may result in even a bigger index. Also, it's best
+   * to call {@link #commit()} afterwards, to allow IndexWriter
+   * to free up disk space.</p>
    *
    * <p>If some but not all readers re-open while merging
    * is underway, this will cause {@code > 2X} temporary

Modified: lucene/dev/branches/lucene_solr_5_1/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_1/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java?rev=1670963&r1=1670962&r2=1670963&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_1/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java (original)
+++ lucene/dev/branches/lucene_solr_5_1/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java Thu Apr  2 20:42:04 2015
@@ -18,12 +18,14 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.Locale;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -146,6 +148,7 @@ public class TestIndexWriterForceMerge e
     if (VERBOSE) {
       System.out.println("TEST: start disk usage = " + startDiskUsage);
     }
+    String startListing = listFiles(dir);
 
     dir.resetMaxUsedSizeInBytes();
     dir.setTrackDiskUsage(true);
@@ -181,11 +184,32 @@ public class TestIndexWriterForceMerge e
     assertTrue("forceMerge used too much temporary space: starting usage was "
         + startDiskUsage + " bytes; final usage was " + finalDiskUsage
         + " bytes; max temp usage was " + maxDiskUsage
-        + " but should have been " + (3 * maxStartFinalDiskUsage)
-        + " (= 3X starting usage)", maxDiskUsage <= 3 * maxStartFinalDiskUsage);
+        + " but should have been at most " + (4 * maxStartFinalDiskUsage)
+        + " (= 4X starting usage), BEFORE=" + startListing + "AFTER=" + listFiles(dir), maxDiskUsage <= 4 * maxStartFinalDiskUsage);
     dir.close();
   }
   
+  // print out listing of files and sizes, but recurse into CFS to debug nested files there.
+  private String listFiles(Directory dir) throws IOException {
+    SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
+    StringBuilder sb = new StringBuilder();
+    sb.append(System.lineSeparator());
+    for (SegmentCommitInfo info : infos) {
+      for (String file : info.files()) {
+        sb.append(String.format(Locale.ROOT, "%-20s%d%n", file, dir.fileLength(file)));
+      }
+      if (info.info.getUseCompoundFile()) {
+        try (Directory cfs = info.info.getCodec().compoundFormat().getCompoundReader(dir, info.info, IOContext.DEFAULT)) {
+          for (String file : cfs.listAll()) {
+            sb.append(String.format(Locale.ROOT, " |- (inside compound file) %-20s%d%n", file, cfs.fileLength(file)));
+          }
+        }
+      }
+    }
+    sb.append(System.lineSeparator());
+    return sb.toString();
+  }
+
   // Test calling forceMerge(1, false) whereby forceMerge is kicked
   // off but we don't wait for it to finish (but
   // writer.close()) does wait