You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/01/29 17:22:59 UTC
svn commit: r1065059 - in
/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index:
ConcurrentMergeScheduler.java SegmentInfo.java
Author: mikemccand
Date: Sat Jan 29 16:22:59 2011
New Revision: 1065059
URL: http://svn.apache.org/viewvc?rev=1065059&view=rev
Log:
LUCENE-2898: fix CMS throttling to be independent of number of incoming producer threads; some defensive concurrency fixes for SegmentInfo
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1065059&r1=1065058&r2=1065059&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Sat Jan 29 16:22:59 2011
@@ -308,10 +308,31 @@ public class ConcurrentMergeScheduler ex
// pending merges, until it's empty:
while (true) {
+ synchronized(this) {
+ long startStallTime = 0;
+ while (mergeThreadCount() >= 1+maxMergeCount) {
+ startStallTime = System.currentTimeMillis();
+ if (verbose()) {
+ message(" too many merges; stalling...");
+ }
+ try {
+ wait();
+ } catch (InterruptedException ie) {
+ throw new ThreadInterruptedException(ie);
+ }
+ }
+
+ if (verbose()) {
+ if (startStallTime != 0) {
+ message(" stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
+ }
+ }
+ }
+
+
// TODO: we could be careful about which merges to do in
// the BG (eg maybe the "biggest" ones) vs FG, which
// merges to do first (the easiest ones?), etc.
-
MergePolicy.OneMerge merge = writer.getNextMerge();
if (merge == null) {
if (verbose())
@@ -326,32 +347,11 @@ public class ConcurrentMergeScheduler ex
boolean success = false;
try {
synchronized(this) {
- final MergeThread merger;
- long startStallTime = 0;
- while (mergeThreadCount() >= maxMergeCount) {
- startStallTime = System.currentTimeMillis();
- if (verbose()) {
- message(" too many merges; stalling...");
- }
- try {
- wait();
- } catch (InterruptedException ie) {
- throw new ThreadInterruptedException(ie);
- }
- }
-
- if (verbose()) {
- if (startStallTime != 0) {
- message(" stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
- }
- message(" consider merge " + merge.segString(dir));
- }
-
- assert mergeThreadCount() < maxMergeCount;
+ message(" consider merge " + merge.segString(dir));
// OK to spawn a new merge thread to handle this
// merge:
- merger = getMergeThread(writer, merge);
+ final MergeThread merger = getMergeThread(writer, merge);
mergeThreads.add(merger);
if (verbose()) {
message(" launch new thread [" + merger.getName() + "]");
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1065059&r1=1065058&r2=1065059&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Sat Jan 29 16:22:59 2011
@@ -66,11 +66,11 @@ public final class SegmentInfo {
private boolean isCompoundFile;
- private List<String> files; // cached list of files that this segment uses
+ private volatile List<String> files; // cached list of files that this segment uses
// in the Directory
- private long sizeInBytesNoStore = -1; // total byte size of all but the store files (computed on demand)
- private long sizeInBytesWithStore = -1; // total byte size of all of our files (computed on demand)
+ private volatile long sizeInBytesNoStore = -1; // total byte size of all but the store files (computed on demand)
+ private volatile long sizeInBytesWithStore = -1; // total byte size of all of our files (computed on demand)
private int docStoreOffset; // if this segment shares stored fields & vectors, this
// offset is where in that file this segment's docs begin
@@ -241,24 +241,31 @@ public final class SegmentInfo {
*/
public long sizeInBytes(boolean includeDocStores) throws IOException {
if (includeDocStores) {
- if (sizeInBytesWithStore != -1) return sizeInBytesWithStore;
- sizeInBytesWithStore = 0;
+ if (sizeInBytesWithStore != -1) {
+ return sizeInBytesWithStore;
+ }
+ long sum = 0;
for (final String fileName : files()) {
- // We don't count bytes used by a shared doc store against this segment
+ // We don't count bytes used by a shared doc store
+ // against this segment
if (docStoreOffset == -1 || !IndexFileNames.isDocStoreFile(fileName)) {
- sizeInBytesWithStore += dir.fileLength(fileName);
+ sum += dir.fileLength(fileName);
}
}
+ sizeInBytesWithStore = sum;
return sizeInBytesWithStore;
} else {
- if (sizeInBytesNoStore != -1) return sizeInBytesNoStore;
- sizeInBytesNoStore = 0;
+ if (sizeInBytesNoStore != -1) {
+ return sizeInBytesNoStore;
+ }
+ long sum = 0;
for (final String fileName : files()) {
if (IndexFileNames.isDocStoreFile(fileName)) {
continue;
}
- sizeInBytesNoStore += dir.fileLength(fileName);
+ sum += dir.fileLength(fileName);
}
+ sizeInBytesNoStore = sum;
return sizeInBytesNoStore;
}
}