You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2011/05/18 19:11:20 UTC
svn commit: r1124339 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/contrib/misc/src/java/org/apache/lucene/index/
lucene/src/java/org/apache/lucene/index/
lucene/src/test-framework/org/apache/lucene/index/ lucene/src/test/org...
Author: uschindler
Date: Wed May 18 17:11:19 2011
New Revision: 1124339
URL: http://svn.apache.org/viewvc?rev=1124339&view=rev
Log:
LUCENE-3084: Eliminate Vector subclassing in SegmentInfos, redesign API.
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Wed May 18 17:11:19 2011
@@ -13,9 +13,15 @@ Changes in backwards compatibility polic
(Mike McCandless, Shai Erera)
* LUCENE-3084: MergePolicy.OneMerge.segments was changed from
- SegmentInfos to a List<SegmentInfo>; this is actually a minor change
- because SegmentInfos itself extends Vector<SegmentInfo>. (Uwe
- Schindler, Mike McCandless)
+ SegmentInfos to a List<SegmentInfo>. SegmentInfos itsself was changed
+ to no longer extend Vector<SegmentInfo> (to update code that is using
+ Vector-API, use the new asList() and asSet() methods returning unmodifiable
+ collections; modifying SegmentInfos is now only possible through
+ the explicitely declared methods). IndexWriter.segString() now takes
+ Iterable<SegmentInfo> instead of List<SegmentInfo>. A simple recompile
+ should fix this. MergePolicy and SegmentInfos are internal/experimental
+ APIs not covered by the strict backwards compatibility policy.
+ (Uwe Schindler, Mike McCandless)
Changes in runtime behavior
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java Wed May 18 17:11:19 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
+import java.util.Collections;
import java.util.Set;
/**
@@ -135,7 +136,7 @@ public class BalancedSegmentMergePolicy
if (last > 1 || !isOptimized(infos.info(0))) {
spec = new MergeSpecification();
- spec.add(new OneMerge(infos.range(0, last)));
+ spec.add(new OneMerge(infos.asList().subList(0, last)));
}
} else if (last > maxNumSegments) {
@@ -192,7 +193,7 @@ public class BalancedSegmentMergePolicy
prev = backLink[i][prev];
int mergeStart = i + prev;
if((mergeEnd - mergeStart) > 1) {
- spec.add(new OneMerge(infos.range(mergeStart, mergeEnd)));
+ spec.add(new OneMerge(infos.asList().subList(mergeStart, mergeEnd)));
} else {
if(partialExpunge) {
SegmentInfo info = infos.info(mergeStart);
@@ -208,7 +209,7 @@ public class BalancedSegmentMergePolicy
if(partialExpunge && maxDelCount > 0) {
// expunge deletes
- spec.add(new OneMerge(infos.range(expungeCandidate, expungeCandidate + 1)));
+ spec.add(new OneMerge(Collections.singletonList(infos.info(expungeCandidate))));
}
return spec;
@@ -250,7 +251,10 @@ public class BalancedSegmentMergePolicy
MergeSpecification spec = null;
if(numLargeSegs < numSegs) {
- SegmentInfos smallSegments = infos.range(numLargeSegs, numSegs);
+ // hack to create a shallow sub-range as SegmentInfos instance,
+ // it does not clone all metadata, but LogMerge does not need it
+ final SegmentInfos smallSegments = new SegmentInfos();
+ smallSegments.rollbackSegmentInfos(infos.asList().subList(numLargeSegs, numSegs));
spec = super.findMergesToExpungeDeletes(smallSegments);
}
@@ -258,7 +262,7 @@ public class BalancedSegmentMergePolicy
for(int i = 0; i < numLargeSegs; i++) {
SegmentInfo info = infos.info(i);
if(info.hasDeletions()) {
- spec.add(new OneMerge(infos.range(i, i + 1)));
+ spec.add(new OneMerge(Collections.singletonList(infos.info(i))));
}
}
return spec;
@@ -296,7 +300,7 @@ public class BalancedSegmentMergePolicy
if(totalSmallSegSize < targetSegSize * 2) {
MergeSpecification spec = findBalancedMerges(infos, numLargeSegs, (numLargeSegs - 1), _partialExpunge);
if(spec == null) spec = new MergeSpecification(); // should not happen
- spec.add(new OneMerge(infos.range(numLargeSegs, numSegs)));
+ spec.add(new OneMerge(infos.asList().subList(numLargeSegs, numSegs)));
return spec;
} else {
return findBalancedMerges(infos, numSegs, numLargeSegs, _partialExpunge);
@@ -311,11 +315,13 @@ public class BalancedSegmentMergePolicy
if(size(info) < sizeThreshold) break;
startSeg++;
}
- spec.add(new OneMerge(infos.range(startSeg, numSegs)));
+ spec.add(new OneMerge(infos.asList().subList(startSeg, numSegs)));
return spec;
} else {
- // apply the log merge policy to small segments.
- SegmentInfos smallSegments = infos.range(numLargeSegs, numSegs);
+ // hack to create a shallow sub-range as SegmentInfos instance,
+ // it does not clone all metadata, but LogMerge does not need it
+ final SegmentInfos smallSegments = new SegmentInfos();
+ smallSegments.rollbackSegmentInfos(infos.asList().subList(numLargeSegs, numSegs));
MergeSpecification spec = super.findMerges(smallSegments);
if(_partialExpunge) {
@@ -342,7 +348,7 @@ public class BalancedSegmentMergePolicy
}
}
if (maxDelCount > 0) {
- return new OneMerge(infos.range(expungeCandidate, expungeCandidate + 1));
+ return new OneMerge(Collections.singletonList(infos.info(expungeCandidate)));
}
return null;
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Wed May 18 17:11:19 2011
@@ -803,8 +803,7 @@ class DirectoryReader extends IndexReade
// case we have to roll back:
startCommit();
- final SegmentInfos rollbackSegmentInfos = new SegmentInfos();
- rollbackSegmentInfos.addAll(segmentInfos);
+ final List<SegmentInfo> rollbackSegments = segmentInfos.createBackupSegmentInfos(false);
boolean success = false;
try {
@@ -836,8 +835,7 @@ class DirectoryReader extends IndexReade
deleter.refresh();
// Restore all SegmentInfos (in case we pruned some)
- segmentInfos.clear();
- segmentInfos.addAll(rollbackSegmentInfos);
+ segmentInfos.rollbackSegmentInfos(rollbackSegments);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed May 18 17:11:19 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -274,7 +275,7 @@ public class IndexWriter implements Clos
private volatile long changeCount; // increments every time a change is completed
private long lastCommitChangeCount; // last changeCount that was committed
- private SegmentInfos rollbackSegmentInfos; // segmentInfos we will fallback to if the commit fails
+ private List<SegmentInfo> rollbackSegments; // list of segmentInfo we will fallback to if the commit fails
volatile SegmentInfos pendingCommit; // set when a commit is pending (after prepareCommit() & before commit())
volatile long pendingCommitChangeCount;
@@ -504,14 +505,14 @@ public class IndexWriter implements Clos
public synchronized boolean infoIsLive(SegmentInfo info) {
int idx = segmentInfos.indexOf(info);
assert idx != -1: "info=" + info + " isn't in pool";
- assert segmentInfos.get(idx) == info: "info=" + info + " doesn't match live info in segmentInfos";
+ assert segmentInfos.info(idx) == info: "info=" + info + " doesn't match live info in segmentInfos";
return true;
}
public synchronized SegmentInfo mapToLive(SegmentInfo info) {
int idx = segmentInfos.indexOf(info);
if (idx != -1) {
- info = segmentInfos.get(idx);
+ info = segmentInfos.info(idx);
}
return info;
}
@@ -1158,7 +1159,7 @@ public class IndexWriter implements Clos
}
}
- setRollbackSegmentInfos(segmentInfos);
+ rollbackSegments = segmentInfos.createBackupSegmentInfos(true);
docWriter = new DocumentsWriter(config, directory, this, getCurrentFieldInfos(), bufferedDeletesStream);
docWriter.setInfoStream(infoStream);
@@ -1241,10 +1242,6 @@ public class IndexWriter implements Clos
return fieldInfos;
}
- private synchronized void setRollbackSegmentInfos(SegmentInfos infos) {
- rollbackSegmentInfos = (SegmentInfos) infos.clone();
- }
-
/**
* Returns the private {@link IndexWriterConfig}, cloned
* from the {@link IndexWriterConfig} passed to
@@ -1925,8 +1922,7 @@ public class IndexWriter implements Clos
else
count = 0;
- for (int i = 0; i < segmentInfos.size(); i++)
- count += segmentInfos.info(i).docCount;
+ count += segmentInfos.totalDocCount();
return count;
}
@@ -1943,8 +1939,7 @@ public class IndexWriter implements Clos
else
count = 0;
- for (int i = 0; i < segmentInfos.size(); i++) {
- final SegmentInfo info = segmentInfos.info(i);
+ for (final SegmentInfo info : segmentInfos) {
count += info.docCount - numDeletedDocs(info);
}
return count;
@@ -1958,9 +1953,11 @@ public class IndexWriter implements Clos
if (docWriter.anyDeletions()) {
return true;
}
- for (int i = 0; i < segmentInfos.size(); i++)
- if (segmentInfos.info(i).hasDeletions())
+ for (final SegmentInfo info : segmentInfos) {
+ if (info.hasDeletions()) {
return true;
+ }
+ }
return false;
}
@@ -2391,7 +2388,8 @@ public class IndexWriter implements Clos
synchronized(this) {
resetMergeExceptions();
- segmentsToOptimize = new HashSet<SegmentInfo>(segmentInfos);
+ segmentsToOptimize.clear();
+ segmentsToOptimize.addAll(segmentInfos.asSet());
optimizeMaxNumSegments = maxNumSegments;
// Now mark all pending & running merges as optimize
@@ -2615,7 +2613,7 @@ public class IndexWriter implements Clos
final MergePolicy.MergeSpecification spec;
if (optimize) {
- spec = mergePolicy.findMergesForOptimize(segmentInfos, maxNumSegmentsOptimize, segmentsToOptimize);
+ spec = mergePolicy.findMergesForOptimize(segmentInfos, maxNumSegmentsOptimize, Collections.unmodifiableSet(segmentsToOptimize));
if (spec != null) {
final int numMerges = spec.merges.size();
@@ -2726,8 +2724,7 @@ public class IndexWriter implements Clos
// attempt to commit using this instance of IndexWriter
// will always write to a new generation ("write
// once").
- segmentInfos.clear();
- segmentInfos.addAll(rollbackSegmentInfos);
+ segmentInfos.rollbackSegmentInfos(rollbackSegments);
docWriter.abort();
@@ -3289,7 +3286,7 @@ public class IndexWriter implements Clos
lastCommitChangeCount = pendingCommitChangeCount;
segmentInfos.updateGeneration(pendingCommit);
segmentInfos.setUserData(pendingCommit.getUserData());
- setRollbackSegmentInfos(pendingCommit);
+ rollbackSegments = pendingCommit.createBackupSegmentInfos(true);
deleter.checkpoint(pendingCommit, true);
} finally {
// Matches the incRef done in startCommit:
@@ -3393,8 +3390,10 @@ public class IndexWriter implements Clos
if (infoStream != null) {
message("apply all deletes during flush");
}
+
flushDeletesCount.incrementAndGet();
- final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, segmentInfos);
+ final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream
+ .applyDeletes(readerPool, segmentInfos.asList());
if (result.anyDeletes) {
checkpoint();
}
@@ -3402,7 +3401,7 @@ public class IndexWriter implements Clos
if (infoStream != null) {
message("drop 100% deleted segments: " + result.allDeleted);
}
- for(SegmentInfo info : result.allDeleted) {
+ for (SegmentInfo info : result.allDeleted) {
// If a merge has already registered for this
// segment, we leave it in the readerPool; the
// merge will skip merging it and will then drop
@@ -3417,6 +3416,7 @@ public class IndexWriter implements Clos
checkpoint();
}
bufferedDeletesStream.prune(segmentInfos);
+
assert !bufferedDeletesStream.any();
flushControl.clearDeletes();
} else if (infoStream != null) {
@@ -3458,7 +3458,7 @@ public class IndexWriter implements Clos
private void ensureValidMerge(MergePolicy.OneMerge merge) throws IOException {
for(SegmentInfo info : merge.segments) {
- if (segmentInfos.indexOf(info) == -1) {
+ if (!segmentInfos.contains(info)) {
throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + segString(), directory);
}
}
@@ -3594,39 +3594,13 @@ public class IndexWriter implements Clos
message("merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert"));
}
- final Set mergedAway = new HashSet<SegmentInfo>(merge.segments);
- int segIdx = 0;
- int newSegIdx = 0;
- boolean inserted = false;
- final int curSegCount = segmentInfos.size();
- while(segIdx < curSegCount) {
- final SegmentInfo info = segmentInfos.info(segIdx++);
- if (mergedAway.contains(info)) {
- if (!inserted && (!allDeleted || keepFullyDeletedSegments)) {
- segmentInfos.set(segIdx-1, merge.info);
- inserted = true;
- newSegIdx++;
- }
- } else {
- segmentInfos.set(newSegIdx++, info);
- }
- }
-
- // Either we found place to insert segment, or, we did
- // not, but only because all segments we merged became
- // deleted while we are merging, in which case it should
- // be the case that the new segment is also all deleted:
- if (!inserted) {
- assert allDeleted;
- if (keepFullyDeletedSegments) {
- segmentInfos.add(0, merge.info);
- } else {
- readerPool.drop(merge.info);
- }
+ final boolean dropSegment = allDeleted && !keepFullyDeletedSegments;
+ segmentInfos.applyMergeChanges(merge, dropSegment);
+
+ if (dropSegment) {
+ readerPool.drop(merge.info);
}
-
- segmentInfos.subList(newSegIdx, segmentInfos.size()).clear();
-
+
if (infoStream != null) {
message("after commit: " + segString());
}
@@ -3761,7 +3735,7 @@ public class IndexWriter implements Clos
if (mergingSegments.contains(info)) {
return false;
}
- if (segmentInfos.indexOf(info) == -1) {
+ if (!segmentInfos.contains(info)) {
return false;
}
if (info.dir != directory) {
@@ -4213,7 +4187,7 @@ public class IndexWriter implements Clos
}
// utility routines for tests
- SegmentInfo newestSegment() {
+ synchronized SegmentInfo newestSegment() {
return segmentInfos.size() > 0 ? segmentInfos.info(segmentInfos.size()-1) : null;
}
@@ -4223,14 +4197,13 @@ public class IndexWriter implements Clos
}
/** @lucene.internal */
- public synchronized String segString(List<SegmentInfo> infos) throws IOException {
- StringBuilder buffer = new StringBuilder();
- final int count = infos.size();
- for(int i = 0; i < count; i++) {
- if (i > 0) {
+ public synchronized String segString(Iterable<SegmentInfo> infos) throws IOException {
+ final StringBuilder buffer = new StringBuilder();
+ for(final SegmentInfo s : infos) {
+ if (buffer.length() > 0) {
buffer.append(' ');
}
- buffer.append(segString(infos.get(i)));
+ buffer.append(segString(s));
}
return buffer.toString();
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Wed May 18 17:11:19 2011
@@ -242,6 +242,7 @@ public abstract class LogMergePolicy ext
private MergeSpecification findMergesForOptimizeSizeLimit(
SegmentInfos infos, int maxNumSegments, int last) throws IOException {
MergeSpecification spec = new MergeSpecification();
+ final List<SegmentInfo> segments = infos.asList();
int start = last - 1;
while (start >= 0) {
@@ -254,12 +255,12 @@ public abstract class LogMergePolicy ext
// unless there is only 1 which is optimized.
if (last - start - 1 > 1 || (start != last - 1 && !isOptimized(infos.info(start + 1)))) {
// there is more than 1 segment to the right of this one, or an unoptimized single segment.
- spec.add(new OneMerge(infos.range(start + 1, last)));
+ spec.add(new OneMerge(segments.subList(start + 1, last)));
}
last = start;
} else if (last - start == mergeFactor) {
// mergeFactor eligible segments were found, add them as a merge.
- spec.add(new OneMerge(infos.range(start, last)));
+ spec.add(new OneMerge(segments.subList(start, last)));
last = start;
}
--start;
@@ -267,7 +268,7 @@ public abstract class LogMergePolicy ext
// Add any left-over segments, unless there is just 1 already optimized.
if (last > 0 && (++start + 1 < last || !isOptimized(infos.info(start)))) {
- spec.add(new OneMerge(infos.range(start, last)));
+ spec.add(new OneMerge(segments.subList(start, last)));
}
return spec.merges.size() == 0 ? null : spec;
@@ -280,11 +281,12 @@ public abstract class LogMergePolicy ext
*/
private MergeSpecification findMergesForOptimizeMaxNumSegments(SegmentInfos infos, int maxNumSegments, int last) throws IOException {
MergeSpecification spec = new MergeSpecification();
+ final List<SegmentInfo> segments = infos.asList();
// First, enroll all "full" merges (size
// mergeFactor) to potentially be run concurrently:
while (last - maxNumSegments + 1 >= mergeFactor) {
- spec.add(new OneMerge(infos.range(last-mergeFactor, last)));
+ spec.add(new OneMerge(segments.subList(last - mergeFactor, last)));
last -= mergeFactor;
}
@@ -296,7 +298,7 @@ public abstract class LogMergePolicy ext
// Since we must optimize down to 1 segment, the
// choice is simple:
if (last > 1 || !isOptimized(infos.info(0))) {
- spec.add(new OneMerge(infos.range(0, last)));
+ spec.add(new OneMerge(segments.subList(0, last)));
}
} else if (last > maxNumSegments) {
@@ -325,7 +327,7 @@ public abstract class LogMergePolicy ext
}
}
- spec.add(new OneMerge(infos.range(bestStart, bestStart+finalMergeSize)));
+ spec.add(new OneMerge(segments.subList(bestStart, bestStart + finalMergeSize)));
}
}
return spec.merges.size() == 0 ? null : spec;
@@ -394,7 +396,8 @@ public abstract class LogMergePolicy ext
@Override
public MergeSpecification findMergesToExpungeDeletes(SegmentInfos segmentInfos)
throws CorruptIndexException, IOException {
- final int numSegments = segmentInfos.size();
+ final List<SegmentInfo> segments = segmentInfos.asList();
+ final int numSegments = segments.size();
if (verbose())
message("findMergesToExpungeDeletes: " + numSegments + " segments");
@@ -416,7 +419,7 @@ public abstract class LogMergePolicy ext
// deletions, so force a merge now:
if (verbose())
message(" add merge " + firstSegmentWithDeletions + " to " + (i-1) + " inclusive");
- spec.add(new OneMerge(segmentInfos.range(firstSegmentWithDeletions, i)));
+ spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, i)));
firstSegmentWithDeletions = i;
}
} else if (firstSegmentWithDeletions != -1) {
@@ -425,7 +428,7 @@ public abstract class LogMergePolicy ext
// mergeFactor segments
if (verbose())
message(" add merge " + firstSegmentWithDeletions + " to " + (i-1) + " inclusive");
- spec.add(new OneMerge(segmentInfos.range(firstSegmentWithDeletions, i)));
+ spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, i)));
firstSegmentWithDeletions = -1;
}
}
@@ -433,7 +436,7 @@ public abstract class LogMergePolicy ext
if (firstSegmentWithDeletions != -1) {
if (verbose())
message(" add merge " + firstSegmentWithDeletions + " to " + (numSegments-1) + " inclusive");
- spec.add(new OneMerge(segmentInfos.range(firstSegmentWithDeletions, numSegments)));
+ spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, numSegments)));
}
return spec;
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java Wed May 18 17:11:19 2011
@@ -84,7 +84,8 @@ public abstract class MergePolicy implem
public OneMerge(List<SegmentInfo> segments) {
if (0 == segments.size())
throw new RuntimeException("segments must include at least one segment");
- this.segments = segments;
+ // clone the list, as the in list may be based off original SegmentInfos and may be modified
+ this.segments = new ArrayList<SegmentInfo>(segments);
int count = 0;
for(SegmentInfo info : segments) {
count += info.docCount;
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Wed May 18 17:11:19 2011
@@ -38,7 +38,7 @@ import java.util.Set;
*
* @lucene.experimental
*/
-public final class SegmentInfo {
+public final class SegmentInfo implements Cloneable {
static final int NO = -1; // e.g. no norms; no deletes;
static final int YES = 1; // e.g. have norms; have deletes;
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Wed May 18 17:11:19 2011
@@ -28,13 +28,16 @@ import org.apache.lucene.util.ThreadInte
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
-import java.util.Vector;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* A collection of segmentInfo objects with methods for operating on
@@ -42,7 +45,7 @@ import java.util.Map;
*
* @lucene.experimental
*/
-public final class SegmentInfos extends Vector<SegmentInfo> {
+public final class SegmentInfos implements Cloneable, Iterable<SegmentInfo> {
/** The file format version, a negative number. */
/* Works since counter, the old 1st entry, is always >= 0 */
@@ -113,7 +116,12 @@ public final class SegmentInfos extends
private Map<String,String> userData = Collections.<String,String>emptyMap(); // Opaque Map<String, String> that user can specify during IndexWriter.commit
private int format;
-
+
+ private List<SegmentInfo> segments = new ArrayList<SegmentInfo>();
+ private Set<SegmentInfo> segmentSet = new HashSet<SegmentInfo>();
+ private transient List<SegmentInfo> cachedUnmodifiableList;
+ private transient Set<SegmentInfo> cachedUnmodifiableSet;
+
/**
* If non-null, information about loading segments_N files
* will be printed here. @see #setInfoStream.
@@ -128,8 +136,8 @@ public final class SegmentInfos extends
return format;
}
- public final SegmentInfo info(int i) {
- return get(i);
+ public SegmentInfo info(int i) {
+ return segments.get(i);
}
/**
@@ -248,7 +256,7 @@ public final class SegmentInfos extends
boolean success = false;
// Clear any previous segments:
- clear();
+ this.clear();
ChecksumIndexInput input = new ChecksumIndexInput(directory.openInput(segmentFileName));
@@ -328,7 +336,7 @@ public final class SegmentInfos extends
if (!success) {
// Clear any segment infos we had loaded so we
// have a clean slate on retry:
- clear();
+ this.clear();
}
}
}
@@ -406,15 +414,14 @@ public final class SegmentInfos extends
/** Prunes any segment whose docs are all deleted. */
public void pruneDeletedSegments() throws IOException {
- int segIdx = 0;
- while(segIdx < size()) {
- final SegmentInfo info = info(segIdx);
+ for(final Iterator<SegmentInfo> it = segments.iterator(); it.hasNext();) {
+ final SegmentInfo info = it.next();
if (info.getDelCount() == info.docCount) {
- remove(segIdx);
- } else {
- segIdx++;
+ it.remove();
+ segmentSet.remove(info);
}
}
+ assert segmentSet.size() == segments.size();
}
/**
@@ -424,12 +431,22 @@ public final class SegmentInfos extends
@Override
public Object clone() {
- SegmentInfos sis = (SegmentInfos) super.clone();
- for(int i=0;i<sis.size();i++) {
- sis.set(i, (SegmentInfo) sis.info(i).clone());
+ try {
+ final SegmentInfos sis = (SegmentInfos) super.clone();
+ // deep clone, first recreate all collections:
+ sis.segments = new ArrayList<SegmentInfo>(size());
+ sis.segmentSet = new HashSet<SegmentInfo>(size());
+ sis.cachedUnmodifiableList = null;
+ sis.cachedUnmodifiableSet = null;
+ for(final SegmentInfo info : this) {
+ // dont directly access segments, use add method!!!
+ sis.add((SegmentInfo) info.clone());
+ }
+ sis.userData = new HashMap<String,String>(userData);
+ return sis;
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException("should not happen", e);
}
- sis.userData = new HashMap<String, String>(userData);
- return sis;
}
/**
@@ -798,10 +815,13 @@ public final class SegmentInfos extends
* instances in the specified range first (inclusive) to
* last (exclusive), so total number of segments returned
* is last-first.
+ * @deprecated use {@code asList().subList(first, last)}
+ * instead.
*/
+ @Deprecated
public SegmentInfos range(int first, int last) {
SegmentInfos infos = new SegmentInfos();
- infos.addAll(super.subList(first, last));
+ infos.addAll(segments.subList(first, last));
return infos;
}
@@ -941,7 +961,7 @@ public final class SegmentInfos extends
finishCommit(dir);
}
- public synchronized String toString(Directory directory) {
+ public String toString(Directory directory) {
StringBuilder buffer = new StringBuilder();
buffer.append(getCurrentSegmentFileName()).append(": ");
final int count = size();
@@ -972,8 +992,7 @@ public final class SegmentInfos extends
* remain write once.
*/
void replace(SegmentInfos other) {
- clear();
- addAll(other);
+ rollbackSegmentInfos(other.asList());
lastGeneration = other.lastGeneration;
}
@@ -992,4 +1011,134 @@ public final class SegmentInfos extends
public void changed() {
version++;
}
+
+ /** applies all changes caused by committing a merge to this SegmentInfos */
+ void applyMergeChanges(MergePolicy.OneMerge merge, boolean dropSegment) {
+ final Set<SegmentInfo> mergedAway = new HashSet<SegmentInfo>(merge.segments);
+ boolean inserted = false;
+ int newSegIdx = 0;
+ for (int segIdx = 0, cnt = segments.size(); segIdx < cnt; segIdx++) {
+ assert segIdx >= newSegIdx;
+ final SegmentInfo info = segments.get(segIdx);
+ if (mergedAway.contains(info)) {
+ if (!inserted && !dropSegment) {
+ segments.set(segIdx, merge.info);
+ inserted = true;
+ newSegIdx++;
+ }
+ } else {
+ segments.set(newSegIdx, info);
+ newSegIdx++;
+ }
+ }
+
+ // Either we found place to insert segment, or, we did
+ // not, but only because all segments we merged became
+ // deleted while we are merging, in which case it should
+ // be the case that the new segment is also all deleted,
+ // we insert it at the beginning if it should not be dropped:
+ if (!inserted && !dropSegment) {
+ segments.add(0, merge.info);
+ }
+
+ // the rest of the segments in list are duplicates, so don't remove from map, only list!
+ segments.subList(newSegIdx, segments.size()).clear();
+
+ // update the Set
+ if (!dropSegment) {
+ segmentSet.add(merge.info);
+ }
+ segmentSet.removeAll(mergedAway);
+
+ assert segmentSet.size() == segments.size();
+ }
+
+ List<SegmentInfo> createBackupSegmentInfos(boolean cloneChildren) {
+ if (cloneChildren) {
+ final List<SegmentInfo> list = new ArrayList<SegmentInfo>(size());
+ for(final SegmentInfo info : this) {
+ list.add((SegmentInfo) info.clone());
+ }
+ return list;
+ } else {
+ return new ArrayList<SegmentInfo>(segments);
+ }
+ }
+
+ void rollbackSegmentInfos(List<SegmentInfo> infos) {
+ this.clear();
+ this.addAll(infos);
+ }
+
+ /** Returns an <b>unmodifiable</b> {@link Iterator} of contained segments in order. */
+ // @Override (comment out until Java 6)
+ public Iterator<SegmentInfo> iterator() {
+ return asList().iterator();
+ }
+
+ /** Returns all contained segments as an <b>unmodifiable</b> {@link List} view. */
+ public List<SegmentInfo> asList() {
+ if (cachedUnmodifiableList == null) {
+ cachedUnmodifiableList = Collections.unmodifiableList(segments);
+ }
+ return cachedUnmodifiableList;
+ }
+
+ /** Returns all contained segments as an <b>unmodifiable</b> {@link Set} view.
+ * The iterator is not sorted, use {@link List} view or {@link #iterator} to get all segments in order. */
+ public Set<SegmentInfo> asSet() {
+ if (cachedUnmodifiableSet == null) {
+ cachedUnmodifiableSet = Collections.unmodifiableSet(segmentSet);
+ }
+ return cachedUnmodifiableSet;
+ }
+
+ public int size() {
+ return segments.size();
+ }
+
+ public void add(SegmentInfo si) {
+ if (segmentSet.contains(si)) {
+ throw new IllegalStateException("Cannot add the same segment two times to this SegmentInfos instance");
+ }
+ segments.add(si);
+ segmentSet.add(si);
+ assert segmentSet.size() == segments.size();
+ }
+
+ public void addAll(Iterable<SegmentInfo> sis) {
+ for (final SegmentInfo si : sis) {
+ this.add(si);
+ }
+ }
+
+ public void clear() {
+ segments.clear();
+ segmentSet.clear();
+ }
+
+ public void remove(SegmentInfo si) {
+ final int index = this.indexOf(si);
+ if (index >= 0) {
+ this.remove(index);
+ }
+ }
+
+ public void remove(int index) {
+ segmentSet.remove(segments.remove(index));
+ assert segmentSet.size() == segments.size();
+ }
+
+ public boolean contains(SegmentInfo si) {
+ return segmentSet.contains(si);
+ }
+
+ public int indexOf(SegmentInfo si) {
+ if (segmentSet.contains(si)) {
+ return segments.indexOf(si);
+ } else {
+ return -1;
+ }
+ }
+
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java Wed May 18 17:11:19 2011
@@ -251,9 +251,7 @@ public class TieredMergePolicy extends M
final Collection<SegmentInfo> merging = writer.get().getMergingSegments();
final Collection<SegmentInfo> toBeMerged = new HashSet<SegmentInfo>();
- final List<SegmentInfo> infosSorted = new ArrayList<SegmentInfo>();
- infosSorted.addAll(infos);
-
+ final List<SegmentInfo> infosSorted = new ArrayList<SegmentInfo>(infos.asList());
Collections.sort(infosSorted, segmentByteSizeDescending);
// Compute total index bytes & print details about the index
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java Wed May 18 17:11:19 2011
@@ -42,14 +42,13 @@ public class MockRandomMergePolicy exten
if (segmentInfos.size() > 1 && random.nextInt(5) == 3) {
- SegmentInfos segmentInfos2 = new SegmentInfos();
- segmentInfos2.addAll(segmentInfos);
- Collections.shuffle(segmentInfos2, random);
+ List<SegmentInfo> segments = new ArrayList<SegmentInfo>(segmentInfos.asList());
+ Collections.shuffle(segments, random);
// TODO: sometimes make more than 1 merge?
mergeSpec = new MergeSpecification();
final int segsToMerge = _TestUtil.nextInt(random, 1, segmentInfos.size());
- mergeSpec.add(new OneMerge(segmentInfos2.range(0, segsToMerge)));
+ mergeSpec.add(new OneMerge(segments.subList(0, segsToMerge)));
}
return mergeSpec;
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java?rev=1124339&r1=1124338&r2=1124339&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java Wed May 18 17:11:19 2011
@@ -126,8 +126,8 @@ public class TestPerSegmentDeletes exten
fsmp.length = 2;
System.out.println("maybeMerge "+writer.segmentInfos);
- SegmentInfo info0 = writer.segmentInfos.get(0);
- SegmentInfo info1 = writer.segmentInfos.get(1);
+ SegmentInfo info0 = writer.segmentInfos.info(0);
+ SegmentInfo info1 = writer.segmentInfos.info(1);
writer.maybeMerge();
System.out.println("maybeMerge after "+writer.segmentInfos);
@@ -197,7 +197,7 @@ public class TestPerSegmentDeletes exten
// deletes for info1, the newly created segment from the
// merge should have no deletes because they were applied in
// the merge
- //SegmentInfo info1 = writer.segmentInfos.get(1);
+ //SegmentInfo info1 = writer.segmentInfos.info(1);
//assertFalse(exists(info1, writer.docWriter.segmentDeletes));
//System.out.println("infos4:"+writer.segmentInfos);
@@ -249,11 +249,7 @@ public class TestPerSegmentDeletes exten
throws CorruptIndexException, IOException {
MergeSpecification ms = new MergeSpecification();
if (doMerge) {
- SegmentInfos mergeInfos = new SegmentInfos();
- for (int x=start; x < (start+length); x++) {
- mergeInfos.add(segmentInfos.get(x));
- }
- OneMerge om = new OneMerge(mergeInfos);
+ OneMerge om = new OneMerge(segmentInfos.asList().subList(start, start + length));
ms.add(om);
doMerge = false;
return ms;