You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/05/05 17:06:15 UTC
svn commit: r1099835 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/src/java/org/apache/lucene/index/LogMergePolicy.java solr/
Author: mikemccand
Date: Thu May 5 15:06:15 2011
New Revision: 1099835
URL: http://svn.apache.org/viewvc?rev=1099835&view=rev
Log:
LUCENE-2904: LogMP pays attention to which segments are already being merged (merged from trunk)
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1099835&r1=1099834&r2=1099835&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Thu May 5 15:06:15 2011
@@ -2618,10 +2618,22 @@ public class IndexWriter implements Clos
}
}
- /**
- * Expert: the {@link MergeScheduler} calls this method to retrieve the next
- * merge requested by the MergePolicy
- *
+ /** Expert: to be used by a {@link MergePolicy} to avoid
+ * selecting merges for segments already being merged.
+ * The returned collection is not cloned, and thus is
+ * only safe to access if you hold IndexWriter's lock
+ * (which you do when IndexWriter invokes the
+ * MergePolicy).
+ *
+ * <p>Do not alter the returned collection! */
+ public synchronized Collection<SegmentInfo> getMergingSegments() {
+ return mergingSegments;
+ }
+
+ /** Expert: the {@link MergeScheduler} calls this method
+ * to retrieve the next merge requested by the
+ * MergePolicy
+ *
* @lucene.experimental
*/
public synchronized MergePolicy.OneMerge getNextMerge() {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1099835&r1=1099834&r2=1099835&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Thu May 5 15:06:15 2011
@@ -18,9 +18,12 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Set;
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
/** <p>This class implements a {@link MergePolicy} that tries
* to merge segments into levels of exponentially
@@ -456,7 +459,7 @@ public abstract class LogMergePolicy ext
return spec;
}
- private static class SegmentInfoAndLevel implements Comparable {
+ private static class SegmentInfoAndLevel implements Comparable<SegmentInfoAndLevel> {
SegmentInfo info;
float level;
int index;
@@ -468,8 +471,7 @@ public abstract class LogMergePolicy ext
}
// Sorts largest to smallest
- public int compareTo(Object o) {
- SegmentInfoAndLevel other = (SegmentInfoAndLevel) o;
+ public int compareTo(SegmentInfoAndLevel other) {
if (level < other.level)
return 1;
else if (level > other.level)
@@ -503,22 +505,37 @@ public abstract class LogMergePolicy ext
// Compute levels, which is just log (base mergeFactor)
// of the size of each segment
- SegmentInfoAndLevel[] levels = new SegmentInfoAndLevel[numSegments];
+ final List<SegmentInfoAndLevel> levels = new ArrayList<SegmentInfoAndLevel>();
final float norm = (float) Math.log(mergeFactor);
+ final Collection<SegmentInfo> mergingSegments = writer.get().getMergingSegments();
+
for(int i=0;i<numSegments;i++) {
final SegmentInfo info = infos.info(i);
long size = size(info);
+ // When we require contiguous merge, we still add the
+ // segment to levels to avoid merging "across" a set
+ // of segment being merged:
+ if (!requireContiguousMerge && mergingSegments.contains(info)) {
+ if (verbose()) {
+ message("seg " + info.name + " already being merged; skip");
+ }
+ continue;
+ }
+
// Floor tiny segments
- if (size < 1)
+ if (size < 1) {
size = 1;
- levels[i] = new SegmentInfoAndLevel(info, (float) Math.log(size)/norm, i);
- message("seg " + info.name + " level=" + levels[i].level + " size=" + size);
+ }
+ levels.add(new SegmentInfoAndLevel(info, (float) Math.log(size)/norm, i));
+ if (verbose()) {
+ message("seg " + info.name + " level=" + levels.get(i).level + " size=" + size);
+ }
}
if (!requireContiguousMerge) {
- Arrays.sort(levels);
+ Collections.sort(levels);
}
final float levelFloor;
@@ -536,14 +553,16 @@ public abstract class LogMergePolicy ext
MergeSpecification spec = null;
+ final int numMergeableSegments = levels.size();
+
int start = 0;
- while(start < numSegments) {
+ while(start < numMergeableSegments) {
// Find max level of all segments not already
// quantized.
- float maxLevel = levels[start].level;
- for(int i=1+start;i<numSegments;i++) {
- final float level = levels[i].level;
+ float maxLevel = levels.get(start).level;
+ for(int i=1+start;i<numMergeableSegments;i++) {
+ final float level = levels.get(i).level;
if (level > maxLevel)
maxLevel = level;
}
@@ -562,9 +581,9 @@ public abstract class LogMergePolicy ext
levelBottom = levelFloor;
}
- int upto = numSegments-1;
+ int upto = numMergeableSegments-1;
while(upto >= start) {
- if (levels[upto].level >= levelBottom) {
+ if (levels.get(upto).level >= levelBottom) {
break;
}
upto--;
@@ -577,7 +596,7 @@ public abstract class LogMergePolicy ext
while(end <= 1+upto) {
boolean anyTooLarge = false;
for(int i=start;i<end;i++) {
- final SegmentInfo info = levels[i].info;
+ final SegmentInfo info = levels.get(i).info;
anyTooLarge |= (size(info) >= maxMergeSize || sizeDocs(info) >= maxMergeDocs);
}
@@ -587,11 +606,11 @@ public abstract class LogMergePolicy ext
if (verbose()) {
message(" " + start + " to " + end + ": add this merge");
}
- Arrays.sort(levels, start, end, sortByIndex);
+ Collections.sort(levels.subList(start, end), sortByIndex);
final SegmentInfos mergeInfos = new SegmentInfos();
for(int i=start;i<end;i++) {
- mergeInfos.add(levels[i].info);
- assert infos.contains(levels[i].info);
+ mergeInfos.add(levels.get(i).info);
+ assert infos.contains(levels.get(i).info);
}
spec.add(new OneMerge(mergeInfos));
} else if (verbose()) {