You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/01/29 18:17:39 UTC

svn commit: r1065077 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java lucene/src/java/org/apache/lucene/index/SegmentInfo.java solr/

Author: mikemccand
Date: Sat Jan 29 17:17:38 2011
New Revision: 1065077

URL: http://svn.apache.org/viewvc?rev=1065077&view=rev
Log:
LUCENE-2898: fix CMS throttling to be independent of number of incoming producer threads; some defensive concurrency fixes for SegmentInfo

Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
    lucene/dev/branches/branch_3x/solr/   (props changed)

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1065077&r1=1065076&r2=1065077&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Sat Jan 29 17:17:38 2011
@@ -317,10 +317,31 @@ public class ConcurrentMergeScheduler ex
     // pending merges, until it's empty:
     while (true) {
 
+      synchronized(this) {
+        long startStallTime = 0;
+        while (mergeThreadCount() >= 1+maxMergeCount) {
+          startStallTime = System.currentTimeMillis();
+          if (verbose()) {
+            message("    too many merges; stalling...");
+          }
+          try {
+            wait();
+          } catch (InterruptedException ie) {
+            throw new ThreadInterruptedException(ie);
+          }
+        }
+
+        if (verbose()) {
+          if (startStallTime != 0) {
+            message("  stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
+          }
+        }
+      }
+
+
       // TODO: we could be careful about which merges to do in
       // the BG (eg maybe the "biggest" ones) vs FG, which
       // merges to do first (the easiest ones?), etc.
-
       MergePolicy.OneMerge merge = writer.getNextMerge();
       if (merge == null) {
         if (verbose())
@@ -335,32 +356,11 @@ public class ConcurrentMergeScheduler ex
       boolean success = false;
       try {
         synchronized(this) {
-          final MergeThread merger;
-          long startStallTime = 0;
-          while (mergeThreadCount() >= maxMergeCount) {
-            startStallTime = System.currentTimeMillis();
-            if (verbose()) {
-              message("    too many merges; stalling...");
-            }
-            try {
-              wait();
-            } catch (InterruptedException ie) {
-              throw new ThreadInterruptedException(ie);
-            }
-          }
-
-          if (verbose()) {
-            if (startStallTime != 0) {
-              message("  stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
-            }
-            message("  consider merge " + merge.segString(dir));
-          }
-
-          assert mergeThreadCount() < maxMergeCount;
+          message("  consider merge " + merge.segString(dir));
 
           // OK to spawn a new merge thread to handle this
           // merge:
-          merger = getMergeThread(writer, merge);
+          final MergeThread merger = getMergeThread(writer, merge);
           mergeThreads.add(merger);
           if (verbose()) {
             message("    launch new thread [" + merger.getName() + "]");

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1065077&r1=1065076&r2=1065077&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Sat Jan 29 17:17:38 2011
@@ -76,11 +76,11 @@ public final class SegmentInfo {
                                                   // and true for newly created merged segments (both
                                                   // compound and non compound).
   
-  private List<String> files;                     // cached list of files that this segment uses
+  private volatile List<String> files;            // cached list of files that this segment uses
                                                   // in the Directory
 
-  private long sizeInBytesNoStore = -1;           // total byte size of all but the store files (computed on demand)
-  private long sizeInBytesWithStore = -1;         // total byte size of all of our files (computed on demand)
+  private volatile long sizeInBytesNoStore = -1;           // total byte size of all but the store files (computed on demand)
+  private volatile long sizeInBytesWithStore = -1;         // total byte size of all of our files (computed on demand)
 
   private int docStoreOffset;                     // if this segment shares stored fields & vectors, this
                                                   // offset is where in that file this segment's docs begin
@@ -288,24 +288,31 @@ public final class SegmentInfo {
    */
   public long sizeInBytes(boolean includeDocStores) throws IOException {
     if (includeDocStores) {
-      if (sizeInBytesWithStore != -1) return sizeInBytesWithStore;
-      sizeInBytesWithStore = 0;
+      if (sizeInBytesWithStore != -1) {
+        return sizeInBytesWithStore;
+      }
+      long sum = 0;
       for (final String fileName : files()) {
-        // We don't count bytes used by a shared doc store against this segment
+        // We don't count bytes used by a shared doc store
+        // against this segment
         if (docStoreOffset == -1 || !IndexFileNames.isDocStoreFile(fileName)) {
-          sizeInBytesWithStore += dir.fileLength(fileName);
+          sum += dir.fileLength(fileName);
         }
       }
+      sizeInBytesWithStore = sum;
       return sizeInBytesWithStore;
     } else {
-      if (sizeInBytesNoStore != -1) return sizeInBytesNoStore;
-      sizeInBytesNoStore = 0;
+      if (sizeInBytesNoStore != -1) {
+        return sizeInBytesNoStore;
+      }
+      long sum = 0;
       for (final String fileName : files()) {
         if (IndexFileNames.isDocStoreFile(fileName)) {
           continue;
         }
-        sizeInBytesNoStore += dir.fileLength(fileName);
+        sum += dir.fileLength(fileName);
       }
+      sizeInBytesNoStore = sum;
       return sizeInBytesNoStore;
     }
   }