You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/05/16 20:40:44 UTC

svn commit: r1595298 - in /lucene/dev/branches/branch_4x/lucene: core/src/java/org/apache/lucene/index/ test-framework/src/java/org/apache/lucene/util/

Author: mikemccand
Date: Fri May 16 18:40:44 2014
New Revision: 1595298

URL: http://svn.apache.org/r1595298
Log:
fix false assert trip; fix LTC to be 'sticky' in how it changes the live IWC settings for flush triggers

Modified:
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1595298&r1=1595297&r2=1595298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Fri May 16 18:40:44 2014
@@ -62,6 +62,7 @@ final class DocumentsWriterFlushControl 
   long peakFlushBytes = 0;// only with assert
   long peakNetBytes = 0;// only with assert
   long peakDelta = 0; // only with assert
+  boolean flushByRAMWasDisabled; // only with assert
   final DocumentsWriterStallControl stallControl;
   private final DocumentsWriterPerThreadPool perThreadPool;
   private final FlushPolicy flushPolicy;
@@ -101,7 +102,9 @@ final class DocumentsWriterFlushControl 
   
   private boolean assertMemory() {
     final double maxRamMB = config.getRAMBufferSizeMB();
-    if (maxRamMB != IndexWriterConfig.DISABLE_AUTO_FLUSH) {
+    // We can only assert if we have always been flushing by RAM usage; otherwise the assert will false trip if e.g. the
+    // flush-by-doc-count * doc size was large enough to use far more RAM than the sudden change to IWC's maxRAMBufferSizeMB:
+    if (maxRamMB != IndexWriterConfig.DISABLE_AUTO_FLUSH && flushByRAMWasDisabled == false) {
       // for this assert we must be tolerant to ram buffer changes!
       maxConfiguredRamBuffer = Math.max(maxRamMB, maxConfiguredRamBuffer);
       final long ram = flushBytes + activeBytes;
@@ -109,7 +112,7 @@ final class DocumentsWriterFlushControl 
       // take peakDelta into account - worst case is that all flushing, pending and blocked DWPT had maxMem and the last doc had the peakDelta
       
       // 2 * ramBufferBytes -> before we stall we need to cross the 2xRAM Buffer border this is still a valid limit
-      // (numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta) -> those are the total number of DWPT that are not active but not yet fully fluhsed
+      // (numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta) -> those are the total number of DWPT that are not active but not yet fully flushed
       // all of them could theoretically be taken out of the loop once they crossed the RAM buffer and the last document was the peak delta
       // (numDocsSinceStalled * peakDelta) -> at any given time there could be n threads in flight that crossed the stall control before we reached the limit and each of them could hold a peak document
       final long expected = (2 * (ramBufferBytes)) + ((numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta) + (numDocsSinceStalled * peakDelta);
@@ -128,8 +131,11 @@ final class DocumentsWriterFlushControl 
             + " byte, flush mem: " + flushBytes + ", active mem: " + activeBytes
             + ", pending DWPT: " + numPending + ", flushing DWPT: "
             + numFlushingDWPT() + ", blocked DWPT: " + numBlockedFlushes()
-            + ", peakDelta mem: " + peakDelta + " byte";
+            + ", peakDelta mem: " + peakDelta + " bytes, ramBufferBytes=" + ramBufferBytes
+            + ", maxConfiguredRamBuffer=" + maxConfiguredRamBuffer;
       }
+    } else {
+      flushByRAMWasDisabled = true;
     }
     return true;
   }
@@ -220,14 +226,14 @@ final class DocumentsWriterFlushControl 
       assert assertMemory();
     } finally {
       try {
-       updateStallState();
+        updateStallState();
       } finally {
         notifyAll();
       }
     }
   }
   
-  private final boolean updateStallState() {
+  private boolean updateStallState() {
     
     assert Thread.holdsLock(this);
     final long limit = stallLimitBytes();

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1595298&r1=1595297&r2=1595298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Fri May 16 18:40:44 2014
@@ -46,7 +46,6 @@ import java.util.Random;
 import java.util.Set;
 import java.util.TimeZone;
 import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -641,6 +640,15 @@ public abstract class LuceneTestCase ext
 
   private static final Map<String,FieldType> fieldToType = new HashMap<String,FieldType>();
 
+  enum LiveIWCFlushMode {BY_RAM, BY_DOCS, EITHER};
+
+  /** Set by TestRuleSetupAndRestoreClassEnv */
+  static LiveIWCFlushMode liveIWCFlushMode;
+
+  static void setLiveIWCFlushMode(LiveIWCFlushMode flushMode) {
+    liveIWCFlushMode = flushMode;
+  }
+
   // -----------------------------------------------------------------
   // Suite and test case setup/ cleanup.
   // -----------------------------------------------------------------
@@ -1058,8 +1066,21 @@ public abstract class LuceneTestCase ext
       // this is complicated because the api requires you "invoke setters in a magical order!"
       // LUCENE-5661: workaround for race conditions in the API
       synchronized (c) {
-        boolean flushByRam = r.nextBoolean();
-        if (flushByRam) { 
+        boolean flushByRAM;
+        switch (liveIWCFlushMode) {
+        case BY_RAM:
+          flushByRAM = true;
+          break;
+        case BY_DOCS:
+          flushByRAM = false;
+          break;
+        case EITHER:
+          flushByRAM = random().nextBoolean();
+          break;
+        default:
+          throw new AssertionError();
+        }
+        if (flushByRAM) { 
           c.setRAMBufferSizeMB(TestUtil.nextInt(r, 1, 10));
           c.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
         } else {

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java?rev=1595298&r1=1595297&r2=1595298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java Fri May 16 18:40:44 2014
@@ -17,17 +17,6 @@ package org.apache.lucene.util;
  * limitations under the License.
  */
 
-import static org.apache.lucene.util.LuceneTestCase.INFOSTREAM;
-import static org.apache.lucene.util.LuceneTestCase.TEST_CODEC;
-import static org.apache.lucene.util.LuceneTestCase.TEST_DOCVALUESFORMAT;
-import static org.apache.lucene.util.LuceneTestCase.TEST_POSTINGSFORMAT;
-import static org.apache.lucene.util.LuceneTestCase.VERBOSE;
-import static org.apache.lucene.util.LuceneTestCase.assumeFalse;
-import static org.apache.lucene.util.LuceneTestCase.localeForName;
-import static org.apache.lucene.util.LuceneTestCase.random;
-import static org.apache.lucene.util.LuceneTestCase.randomLocale;
-import static org.apache.lucene.util.LuceneTestCase.randomTimeZone;
-
 import java.io.PrintStream;
 import java.util.Arrays;
 import java.util.Date;
@@ -43,16 +32,16 @@ import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.asserting.AssertingCodec;
-import org.apache.lucene.codecs.lucene3x.PreFlexRWCodec;
 import org.apache.lucene.codecs.cheapbastard.CheapBastardCodec;
 import org.apache.lucene.codecs.compressing.CompressingCodec;
+import org.apache.lucene.codecs.lucene3x.PreFlexRWCodec;
 import org.apache.lucene.codecs.lucene40.Lucene40RWCodec;
 import org.apache.lucene.codecs.lucene40.Lucene40RWPostingsFormat;
 import org.apache.lucene.codecs.lucene41.Lucene41RWCodec;
-import org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat;
 import org.apache.lucene.codecs.lucene42.Lucene42RWCodec;
 import org.apache.lucene.codecs.lucene45.Lucene45RWCodec;
 import org.apache.lucene.codecs.lucene46.Lucene46Codec;
+import org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat;
 import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
 import org.apache.lucene.index.RandomCodec;
 import org.apache.lucene.search.RandomSimilarityProvider;
@@ -60,9 +49,20 @@ import org.apache.lucene.search.similari
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.junit.internal.AssumptionViolatedException;
-
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 
+import static org.apache.lucene.util.LuceneTestCase.INFOSTREAM;
+import static org.apache.lucene.util.LuceneTestCase.LiveIWCFlushMode;
+import static org.apache.lucene.util.LuceneTestCase.TEST_CODEC;
+import static org.apache.lucene.util.LuceneTestCase.TEST_DOCVALUESFORMAT;
+import static org.apache.lucene.util.LuceneTestCase.TEST_POSTINGSFORMAT;
+import static org.apache.lucene.util.LuceneTestCase.VERBOSE;
+import static org.apache.lucene.util.LuceneTestCase.assumeFalse;
+import static org.apache.lucene.util.LuceneTestCase.localeForName;
+import static org.apache.lucene.util.LuceneTestCase.random;
+import static org.apache.lucene.util.LuceneTestCase.randomLocale;
+import static org.apache.lucene.util.LuceneTestCase.randomTimeZone;
+
 /**
  * Setup and restore suite-level environment (fine grained junk that 
  * doesn't fit anywhere else).
@@ -280,6 +280,25 @@ final class TestRuleSetupAndRestoreClass
           Arrays.toString(avoidCodecs.toArray()));
       throw e;
     }
+
+    // We have "stickiness" so that sometimes all we do is vary the RAM buffer size, other times just the doc count to flush by, else both.
+    // This way the assertMemory in DocumentsWriterFlushControl sometimes runs (when we always flush by RAM).
+    LiveIWCFlushMode flushMode;
+    switch (random().nextInt(3)) {
+    case 0:
+      flushMode = LiveIWCFlushMode.BY_RAM;
+      break;
+    case 1:
+      flushMode = LiveIWCFlushMode.BY_DOCS;
+      break;
+    case 2:
+      flushMode = LiveIWCFlushMode.EITHER;
+      break;
+    default:
+      throw new AssertionError();
+    }
+
+    LuceneTestCase.setLiveIWCFlushMode(flushMode);
   }
 
   /**