You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/12/08 19:35:42 UTC
svn commit: r1043582 - in /lucene/dev/branches/docvalues/lucene: ./ contrib/
contrib/misc/src/java/org/apache/lucene/index/
contrib/misc/src/java/org/apache/lucene/store/
contrib/misc/src/test/org/apache/lucene/index/
src/java/org/apache/lucene/index/ ...
Author: simonw
Date: Wed Dec 8 18:35:41 2010
New Revision: 1043582
URL: http://svn.apache.org/viewvc?rev=1043582&view=rev
Log:
catch up with trunk
Added:
lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp
- copied unchanged from r1043147, lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp
lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
- copied unchanged from r1043147, lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java
- copied unchanged from r1043147, lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java
Modified:
lucene/dev/branches/docvalues/lucene/ (props changed)
lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt
lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp
lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java
Modified: lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt Wed Dec 8 18:35:41 2010
@@ -32,6 +32,11 @@ New Features
* LUCENE-2507: Added DirectSpellChecker, which retrieves correction candidates directly
from the term dictionary using levenshtein automata. (Robert Muir)
+ * LUCENE-2791: Added WindowsDirectory, a Windows-specific Directory impl
+ that doesn't synchronize on the file handle. This can be useful to
+ avoid the performance problems of SimpleFSDirectory and NIOFSDirectory.
+ (Robert Muir, Simon Willnauer, Uwe Schindler, Michael McCandless)
+
API Changes
* LUCENE-2606: Changed RegexCapabilities interface to fix thread
@@ -166,6 +171,9 @@ API Changes
new SpanMultiTermQueryWrapper<RegexQuery>(new RegexQuery()) instead.
(Robert Muir, Uwe Schindler)
+ * LUCENE-2747: Deprecated ArabicLetterTokenizer. StandardTokenizer now tokenizes
+ most languages correctly including Arabic. (Steven Rowe, Robert Muir)
+
New features
* LUCENE-2306: Add NumericRangeFilter and NumericRangeQuery support to XMLQueryParser.
@@ -274,7 +282,7 @@ Build
dependency management between contribs by a new ANT macro.
(Uwe Schindler, Shai Erera)
- * LUCENE-2399, LUCENE-2683: Upgrade contrib/icu's ICU jar file to ICU 4.4.2
+ * LUCENE-2797: Upgrade contrib/icu's ICU jar file to ICU 4.6
(Robert Muir)
Optimizations
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java Wed Dec 8 18:35:41 2010
@@ -132,11 +132,10 @@ public class BalancedSegmentMergePolicy
// Since we must optimize down to 1 segment, the
// choice is simple:
- boolean useCompoundFile = getUseCompoundFile();
if (last > 1 || !isOptimized(infos.info(0))) {
spec = new MergeSpecification();
- spec.add(new OneMerge(infos.range(0, last), useCompoundFile));
+ spec.add(new OneMerge(infos.range(0, last)));
}
} else if (last > maxNumSegments) {
@@ -153,7 +152,6 @@ public class BalancedSegmentMergePolicy
if (infoLen <= maxNumSegments) return null;
MergeSpecification spec = new MergeSpecification();
- boolean useCompoundFile = getUseCompoundFile();
// use Viterbi algorithm to find the best segmentation.
// we will try to minimize the size variance of resulting segments.
@@ -194,7 +192,7 @@ public class BalancedSegmentMergePolicy
prev = backLink[i][prev];
int mergeStart = i + prev;
if((mergeEnd - mergeStart) > 1) {
- spec.add(new OneMerge(infos.range(mergeStart, mergeEnd), useCompoundFile));
+ spec.add(new OneMerge(infos.range(mergeStart, mergeEnd)));
} else {
if(partialExpunge) {
SegmentInfo info = infos.info(mergeStart);
@@ -210,7 +208,7 @@ public class BalancedSegmentMergePolicy
if(partialExpunge && maxDelCount > 0) {
// expunge deletes
- spec.add(new OneMerge(infos.range(expungeCandidate, expungeCandidate + 1), useCompoundFile));
+ spec.add(new OneMerge(infos.range(expungeCandidate, expungeCandidate + 1)));
}
return spec;
@@ -260,7 +258,7 @@ public class BalancedSegmentMergePolicy
for(int i = 0; i < numLargeSegs; i++) {
SegmentInfo info = infos.info(i);
if(info.hasDeletions()) {
- spec.add(new OneMerge(infos.range(i, i + 1), getUseCompoundFile()));
+ spec.add(new OneMerge(infos.range(i, i + 1)));
}
}
return spec;
@@ -298,7 +296,7 @@ public class BalancedSegmentMergePolicy
if(totalSmallSegSize < targetSegSize * 2) {
MergeSpecification spec = findBalancedMerges(infos, numLargeSegs, (numLargeSegs - 1), _partialExpunge);
if(spec == null) spec = new MergeSpecification(); // should not happen
- spec.add(new OneMerge(infos.range(numLargeSegs, numSegs), getUseCompoundFile()));
+ spec.add(new OneMerge(infos.range(numLargeSegs, numSegs)));
return spec;
} else {
return findBalancedMerges(infos, numSegs, numLargeSegs, _partialExpunge);
@@ -313,7 +311,7 @@ public class BalancedSegmentMergePolicy
if(size(info) < sizeThreshold) break;
startSeg++;
}
- spec.add(new OneMerge(infos.range(startSeg, numSegs), getUseCompoundFile()));
+ spec.add(new OneMerge(infos.range(startSeg, numSegs)));
return spec;
} else {
// apply the log merge policy to small segments.
@@ -344,7 +342,7 @@ public class BalancedSegmentMergePolicy
}
}
if (maxDelCount > 0) {
- return new OneMerge(infos.range(expungeCandidate, expungeCandidate + 1), getUseCompoundFile());
+ return new OneMerge(infos.range(expungeCandidate, expungeCandidate + 1));
}
return null;
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp Wed Dec 8 18:35:41 2010
@@ -97,9 +97,9 @@ JNIEXPORT jobject JNICALL Java_org_apach
fname = (char *) env->GetStringUTFChars(filename, NULL);
if (readOnly) {
- fd = open(fname, O_RDONLY | O_DIRECT);
+ fd = open(fname, O_RDONLY | O_DIRECT | O_NOATIME);
} else {
- fd = open(fname, O_RDWR | O_CREAT | O_DIRECT, 0666);
+ fd = open(fname, O_RDWR | O_CREAT | O_DIRECT | O_NOATIME, 0666);
}
//printf("open %s -> %d; ro %d\n", fname, fd, readOnly); fflush(stdout);
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java Wed Dec 8 18:35:41 2010
@@ -34,7 +34,15 @@ public class TestIndexSplitter extends L
_TestUtil.rmDir(destDir);
destDir.mkdirs();
FSDirectory fsDir = FSDirectory.open(dir);
- IndexWriter iw = new IndexWriter(fsDir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE));
+
+ LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
+ mergePolicy.setNoCFSRatio(1);
+ IndexWriter iw = new IndexWriter(
+ fsDir,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
+ setOpenMode(OpenMode.CREATE).
+ setMergePolicy(mergePolicy)
+ );
for (int x=0; x < 100; x++) {
Document doc = TestIndexWriterReader.createDocument(x, "index", 5);
iw.addDocument(doc);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed Dec 8 18:35:41 2010
@@ -50,13 +50,13 @@ import java.util.Date;
An <code>IndexWriter</code> creates and maintains an index.
<p>The <code>create</code> argument to the {@link
- #IndexWriter(Directory, Analyzer, boolean, MaxFieldLength) constructor} determines
+ #IndexWriter(Directory, IndexWriterConfig) constructor} determines
whether a new index is created, or whether an existing index is
opened. Note that you can open an index with <code>create=true</code>
even while readers are using the index. The old readers will
continue to search the "point in time" snapshot they had opened,
and won't see the newly created index until they re-open. There are
- also {@link #IndexWriter(Directory, Analyzer, MaxFieldLength) constructors}
+ also {@link #IndexWriter(Directory, IndexWriterConfig) constructors}
with no <code>create</code> argument which will create a new index
if there is not already an index at the provided path and otherwise
open the existing index.</p>
@@ -72,11 +72,11 @@ import java.util.Date;
<p>These changes are buffered in memory and periodically
flushed to the {@link Directory} (during the above method
calls). A flush is triggered when there are enough
- buffered deletes (see {@link #setMaxBufferedDeleteTerms})
+ buffered deletes (see {@link IndexWriterConfig#setMaxBufferedDeleteTerms})
or enough added documents since the last flush, whichever
is sooner. For the added documents, flushing is triggered
either by RAM usage of the documents (see {@link
- #setRAMBufferSizeMB}) or the number of added documents.
+ IndexWriterConfig#setRAMBufferSizeMB}) or the number of added documents.
The default is to flush when RAM usage hits 16 MB. For
best indexing speed you should flush by RAM usage with a
large RAM buffer. Note that flushing just moves the
@@ -1252,8 +1252,8 @@ public class IndexWriter implements Clos
/**
* Adds a document to this index. If the document contains more than
- * {@link #setMaxFieldLength(int)} terms for a given field, the remainder are
- * discarded.
+ * {@link IndexWriterConfig#setMaxFieldLength(int)} terms for a given field,
+ * the remainder are discarded.
*
* <p> Note that if an Exception is hit (for example disk full)
* then the index will be consistent, but this document
@@ -1301,7 +1301,7 @@ public class IndexWriter implements Clos
/**
* Adds a document to this index, using the provided analyzer instead of the
* value of {@link #getAnalyzer()}. If the document contains more than
- * {@link #setMaxFieldLength(int)} terms for a given field, the remainder are
+ * {@link IndexWriterConfig#setMaxFieldLength(int)} terms for a given field, the remainder are
* discarded.
*
* <p>See {@link #addDocument(Document)} for details on
@@ -1608,7 +1608,7 @@ public class IndexWriter implements Clos
*
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
- * @see LogMergePolicy#findMergesForOptimize
+ * @see MergePolicy#findMergesForOptimize
*/
public void optimize() throws CorruptIndexException, IOException {
optimize(true);
@@ -2289,8 +2289,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void addIndexes(IndexReader... readers)
- throws CorruptIndexException, IOException {
+ public void addIndexes(IndexReader... readers) throws CorruptIndexException, IOException {
ensureOpen();
try {
@@ -2303,47 +2302,33 @@ public class IndexWriter implements Clos
int docCount = merger.merge(); // merge 'em
- SegmentInfo info = null;
+ SegmentInfo info = new SegmentInfo(mergedName, docCount, directory,
+ false, -1, null, false, merger.hasProx(), merger.getSegmentCodecs());
+ setDiagnostics(info, "addIndexes(IndexReader...)");
+
+ boolean useCompoundFile;
+ synchronized(this) { // Guard segmentInfos
+ useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, info);
+ }
+
+ // Now create the compound file if needed
+ if (useCompoundFile) {
+ merger.createCompoundFile(mergedName + ".cfs", info);
+ info.setUseCompoundFile(true);
+
+ // delete new non cfs files directly: they were never
+ // registered with IFD
+ deleter.deleteNewFiles(merger.getMergedFiles(info));
+ }
+
+ // Register the new segment
synchronized(this) {
- info = new SegmentInfo(mergedName, docCount, directory, false, -1,
- null, false, merger.hasProx(), merger.getSegmentCodecs());
- setDiagnostics(info, "addIndexes(IndexReader...)");
segmentInfos.add(info);
- checkpoint();
// Notify DocumentsWriter that the flushed count just increased
docWriter.updateFlushedDocCount(docCount);
- }
-
- // Now create the compound file if needed
- if (mergePolicy instanceof LogMergePolicy && ((LogMergePolicy) mergePolicy).getUseCompoundFile()) {
-
- List<String> files = null;
-
- synchronized(this) {
- // Must incRef our files so that if another thread
- // is running merge/optimize, it doesn't delete our
- // segment's files before we have a chance to
- // finish making the compound file.
- if (segmentInfos.contains(info)) {
- files = info.files();
- deleter.incRef(files);
- }
- }
-
- if (files != null) {
- try {
- merger.createCompoundFile(mergedName + ".cfs", info);
- synchronized(this) {
- info.setUseCompoundFile(true);
- checkpoint();
- }
- } finally {
- synchronized(this) {
- deleter.decRef(files);
- }
- }
- }
+
+ checkpoint();
}
} catch (OutOfMemoryError oom) {
handleOOM(oom, "addIndexes(IndexReader...)");
@@ -3447,8 +3432,12 @@ public class IndexWriter implements Clos
//System.out.println("merger set hasProx=" + merger.hasProx() + " seg=" + merge.info.name);
merge.info.setHasProx(merger.hasProx());
- if (merge.useCompoundFile) {
+ boolean useCompoundFile;
+ synchronized (this) { // Guard segmentInfos
+ useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, merge.info);
+ }
+ if (useCompoundFile) {
success = false;
final String compoundFileName = IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Wed Dec 8 18:35:41 2010
@@ -127,8 +127,21 @@ public abstract class LogMergePolicy ext
// Javadoc inherited
@Override
- public boolean useCompoundFile(SegmentInfos infos, SegmentInfo info) {
- return useCompoundFile;
+ public boolean useCompoundFile(SegmentInfos infos, SegmentInfo mergedInfo) throws IOException {
+ final boolean doCFS;
+
+ if (!useCompoundFile) {
+ doCFS = false;
+ } else if (noCFSRatio == 1.0) {
+ doCFS = true;
+ } else {
+ long totalSize = 0;
+ for (SegmentInfo info : infos)
+ totalSize += size(info);
+
+ doCFS = size(mergedInfo) <= noCFSRatio * totalSize;
+ }
+ return doCFS;
}
/** Sets whether compound file format should be used for
@@ -254,12 +267,12 @@ public abstract class LogMergePolicy ext
// unless there is only 1 which is optimized.
if (last - start - 1 > 1 || (start != last - 1 && !isOptimized(infos.info(start + 1)))) {
// there is more than 1 segment to the right of this one, or an unoptimized single segment.
- spec.add(makeOneMerge(infos, infos.range(start + 1, last)));
+ spec.add(new OneMerge(infos.range(start + 1, last)));
}
last = start;
} else if (last - start == mergeFactor) {
// mergeFactor eligible segments were found, add them as a merge.
- spec.add(makeOneMerge(infos, infos.range(start, last)));
+ spec.add(new OneMerge(infos.range(start, last)));
last = start;
}
--start;
@@ -267,7 +280,7 @@ public abstract class LogMergePolicy ext
// Add any left-over segments, unless there is just 1 already optimized.
if (last > 0 && (++start + 1 < last || !isOptimized(infos.info(start)))) {
- spec.add(makeOneMerge(infos, infos.range(start, last)));
+ spec.add(new OneMerge(infos.range(start, last)));
}
return spec.merges.size() == 0 ? null : spec;
@@ -284,7 +297,7 @@ public abstract class LogMergePolicy ext
// First, enroll all "full" merges (size
// mergeFactor) to potentially be run concurrently:
while (last - maxNumSegments + 1 >= mergeFactor) {
- spec.add(makeOneMerge(infos, infos.range(last-mergeFactor, last)));
+ spec.add(new OneMerge(infos.range(last - mergeFactor, last)));
last -= mergeFactor;
}
@@ -296,7 +309,7 @@ public abstract class LogMergePolicy ext
// Since we must optimize down to 1 segment, the
// choice is simple:
if (last > 1 || !isOptimized(infos.info(0))) {
- spec.add(makeOneMerge(infos, infos.range(0, last)));
+ spec.add(new OneMerge(infos.range(0, last)));
}
} else if (last > maxNumSegments) {
@@ -325,7 +338,7 @@ public abstract class LogMergePolicy ext
}
}
- spec.add(makeOneMerge(infos, infos.range(bestStart, bestStart+finalMergeSize)));
+ spec.add(new OneMerge(infos.range(bestStart, bestStart + finalMergeSize)));
}
}
return spec.merges.size() == 0 ? null : spec;
@@ -413,7 +426,7 @@ public abstract class LogMergePolicy ext
// deletions, so force a merge now:
if (verbose())
message(" add merge " + firstSegmentWithDeletions + " to " + (i-1) + " inclusive");
- spec.add(makeOneMerge(segmentInfos, segmentInfos.range(firstSegmentWithDeletions, i)));
+ spec.add(new OneMerge(segmentInfos.range(firstSegmentWithDeletions, i)));
firstSegmentWithDeletions = i;
}
} else if (firstSegmentWithDeletions != -1) {
@@ -422,7 +435,7 @@ public abstract class LogMergePolicy ext
// mergeFactor segments
if (verbose())
message(" add merge " + firstSegmentWithDeletions + " to " + (i-1) + " inclusive");
- spec.add(makeOneMerge(segmentInfos, segmentInfos.range(firstSegmentWithDeletions, i)));
+ spec.add(new OneMerge(segmentInfos.range(firstSegmentWithDeletions, i)));
firstSegmentWithDeletions = -1;
}
}
@@ -430,7 +443,7 @@ public abstract class LogMergePolicy ext
if (firstSegmentWithDeletions != -1) {
if (verbose())
message(" add merge " + firstSegmentWithDeletions + " to " + (numSegments-1) + " inclusive");
- spec.add(makeOneMerge(segmentInfos, segmentInfos.range(firstSegmentWithDeletions, numSegments)));
+ spec.add(new OneMerge(segmentInfos.range(firstSegmentWithDeletions, numSegments)));
}
return spec;
@@ -530,7 +543,7 @@ public abstract class LogMergePolicy ext
spec = new MergeSpecification();
if (verbose())
message(" " + start + " to " + end + ": add this merge");
- spec.add(makeOneMerge(infos, infos.range(start, end)));
+ spec.add(new OneMerge(infos.range(start, end)));
} else if (verbose())
message(" " + start + " to " + end + ": contains segment over maxMergeSize or maxMergeDocs; skipping");
@@ -544,29 +557,6 @@ public abstract class LogMergePolicy ext
return spec;
}
- protected OneMerge makeOneMerge(SegmentInfos infos, SegmentInfos infosToMerge) throws IOException {
- final boolean doCFS;
- if (!useCompoundFile) {
- doCFS = false;
- } else if (noCFSRatio == 1.0) {
- doCFS = true;
- } else {
-
- long totSize = 0;
- for(SegmentInfo info : infos) {
- totSize += size(info);
- }
- long mergeSize = 0;
- for(SegmentInfo info : infosToMerge) {
- mergeSize += size(info);
- }
-
- doCFS = mergeSize <= noCFSRatio * totSize;
- }
-
- return new OneMerge(infosToMerge, doCFS);
- }
-
/** <p>Determines the largest segment (measured by
* document count) that may be merged with other segments.
* Small values (e.g., less than 10,000) are best for
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MergePolicy.java Wed Dec 8 18:35:41 2010
@@ -76,16 +76,14 @@ public abstract class MergePolicy implem
SegmentReader[] readers; // used by IndexWriter
SegmentReader[] readersClone; // used by IndexWriter
public final SegmentInfos segments;
- public final boolean useCompoundFile;
boolean aborted;
Throwable error;
boolean paused;
- public OneMerge(SegmentInfos segments, boolean useCompoundFile) {
+ public OneMerge(SegmentInfos segments) {
if (0 == segments.size())
throw new RuntimeException("segments must include at least one segment");
this.segments = segments;
- this.useCompoundFile = useCompoundFile;
}
/** Record that an exception occurred while executing
@@ -314,10 +312,9 @@ public abstract class MergePolicy implem
public abstract void close();
/**
- * Returns true if a newly flushed (not from merge)
- * segment should use the compound file format.
+ * Returns true if a new segment (regardless of its origin) should use the compound file format.
*/
- public abstract boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment);
+ public abstract boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment) throws IOException;
/**
* Returns true if the doc store files should use the
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java Wed Dec 8 18:35:41 2010
@@ -110,7 +110,6 @@ public class ByteValuesCreator extends C
vals.values = new byte[maxDoc];
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@@ -120,7 +119,7 @@ public class ByteValuesCreator extends C
break;
}
final byte termval = parser.parseByte(term);
- docs = termsEnum.docs(delDocs, docs);
+ docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -137,7 +136,7 @@ public class ByteValuesCreator extends C
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
- vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
+ vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}
if( vals.valid == null && vals.numDocs < 1 ) {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java Wed Dec 8 18:35:41 2010
@@ -100,22 +100,13 @@ public abstract class CachedArrayCreator
/**
* Utility function to help check what bits are valid
*/
- protected Bits checkMatchAllBits( Bits deleted, OpenBitSet valid, int numDocs, int maxDocs )
+ protected Bits checkMatchAllBits( OpenBitSet valid, int numDocs, int maxDocs )
{
if( numDocs != maxDocs ) {
if( hasOption( OPTION_CACHE_BITS ) ) {
- if( deleted == null ) {
- for( int i=0; i<maxDocs; i++ ) {
- if( !valid.get(i) ) {
- return valid;
- }
- }
- }
- else {
- for( int i=0; i<maxDocs; i++ ) {
- if( !deleted.get(i) && !valid.get(i) ) {
- return valid;
- }
+ for( int i=0; i<maxDocs; i++ ) {
+ if( !valid.get(i) ) {
+ return valid;
}
}
}
@@ -132,7 +123,6 @@ public abstract class CachedArrayCreator
Terms terms = MultiFields.getTerms(reader, field);
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = new OpenBitSet( reader.maxDoc() );
DocsEnum docs = null;
while(true) {
@@ -140,7 +130,7 @@ public abstract class CachedArrayCreator
if (term == null) {
break;
}
- docs = termsEnum.docs(delDocs, docs);
+ docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -152,7 +142,7 @@ public abstract class CachedArrayCreator
vals.numTerms++;
}
- vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, reader.maxDoc() );
+ vals.valid = checkMatchAllBits( validBits, vals.numDocs, reader.maxDoc() );
}
if( vals.numDocs < 1 ) {
vals.valid = new Bits.MatchNoBits( reader.maxDoc() );
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java Wed Dec 8 18:35:41 2010
@@ -130,7 +130,6 @@ public class DocTermsIndexCreator extend
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
DocsEnum docs = null;
while(true) {
@@ -149,7 +148,7 @@ public class DocTermsIndexCreator extend
termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
}
termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
- docs = termsEnum.docs(delDocs, docs);
+ docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java Wed Dec 8 18:35:41 2010
@@ -120,7 +120,6 @@ public class DoubleValuesCreator extends
vals.values = null;
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@@ -130,7 +129,7 @@ public class DoubleValuesCreator extends
break;
}
final double termval = parser.parseDouble(term);
- docs = termsEnum.docs(delDocs, docs);
+ docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -150,7 +149,7 @@ public class DoubleValuesCreator extends
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
- vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
+ vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java Wed Dec 8 18:35:41 2010
@@ -121,7 +121,6 @@ public class FloatValuesCreator extends
vals.values = null;
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@@ -131,7 +130,7 @@ public class FloatValuesCreator extends
break;
}
final float termval = parser.parseFloat(term);
- docs = termsEnum.docs(delDocs, docs);
+ docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -151,7 +150,7 @@ public class FloatValuesCreator extends
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
- vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
+ vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java Wed Dec 8 18:35:41 2010
@@ -121,7 +121,6 @@ public class IntValuesCreator extends Ca
vals.values = null;
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@@ -131,7 +130,7 @@ public class IntValuesCreator extends Ca
break;
}
final int termval = parser.parseInt(term);
- docs = termsEnum.docs(delDocs, docs);
+ docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -151,7 +150,7 @@ public class IntValuesCreator extends Ca
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
- vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
+ vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java Wed Dec 8 18:35:41 2010
@@ -121,7 +121,6 @@ public class LongValuesCreator extends C
vals.values = null;
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@@ -131,7 +130,7 @@ public class LongValuesCreator extends C
break;
}
final long termval = parser.parseLong(term);
- docs = termsEnum.docs(delDocs, docs);
+ docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -151,7 +150,7 @@ public class LongValuesCreator extends C
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
- vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
+ vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java Wed Dec 8 18:35:41 2010
@@ -111,7 +111,6 @@ public class ShortValuesCreator extends
vals.values = new short[maxDoc];
if (terms != null) {
final TermsEnum termsEnum = terms.iterator();
- final Bits delDocs = MultiFields.getDeletedDocs(reader);
OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null;
DocsEnum docs = null;
try {
@@ -121,7 +120,7 @@ public class ShortValuesCreator extends
break;
}
final Short termval = parser.parseShort(term);
- docs = termsEnum.docs(delDocs, docs);
+ docs = termsEnum.docs(null, docs);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -138,7 +137,7 @@ public class ShortValuesCreator extends
} catch (FieldCache.StopFillCacheException stop) {}
if( vals.valid == null ) {
- vals.valid = checkMatchAllBits( delDocs, validBits, vals.numDocs, maxDoc );
+ vals.valid = checkMatchAllBits( validBits, vals.numDocs, maxDoc );
}
}
if( vals.valid == null && vals.numDocs < 1 ) {
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Wed Dec 8 18:35:41 2010
@@ -24,6 +24,9 @@ import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
@@ -36,6 +39,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Version;
import org.apache.lucene.util._TestUtil;
public class TestAddIndexes extends LuceneTestCase {
@@ -1014,5 +1018,30 @@ public class TestAddIndexes extends Luce
setFieldCodec("content", mockSepCodec.name);
}
}
+
+ // LUCENE-2790: tests that the non CFS files were deleted by addIndexes
+ public void testNonCFSLeftovers() throws Exception {
+ Directory[] dirs = new Directory[2];
+ for (int i = 0; i < dirs.length; i++) {
+ dirs[i] = new RAMDirectory();
+ IndexWriter w = new IndexWriter(dirs[i], new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document d = new Document();
+ d.add(new Field("c", "v", Store.YES, Index.ANALYZED, TermVector.YES));
+ w.addDocument(d);
+ w.close();
+ }
+
+ IndexReader[] readers = new IndexReader[] { IndexReader.open(dirs[0]), IndexReader.open(dirs[1]) };
+
+ Directory dir = new RAMDirectory();
+ IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
+ LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
+ lmp.setNoCFSRatio(1.0); // Force creation of CFS
+ IndexWriter w3 = new IndexWriter(dir, conf);
+ w3.addIndexes(readers);
+ w3.close();
+
+ assertEquals("Only one compound segment should exist", 3, dir.listAll().length);
+ }
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Wed Dec 8 18:35:41 2010
@@ -527,12 +527,15 @@ public class TestBackwardsCompatibility
try {
Directory dir = FSDirectory.open(new File(fullDir(outputDir)));
+ LogMergePolicy mergePolicy = newLogMergePolicy(true, 10);
+ mergePolicy.setNoCFSRatio(1); // This test expects all of its segments to be in CFS
+
IndexWriter writer = new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
setMaxBufferedDocs(-1).
setRAMBufferSizeMB(16.0).
- setMergePolicy(newLogMergePolicy(true, 10))
+ setMergePolicy(mergePolicy)
);
for(int i=0;i<35;i++) {
addDoc(writer, i);
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Wed Dec 8 18:35:41 2010
@@ -40,18 +40,23 @@ public class TestIndexFileDeleter extend
public void testDeleteLeftoverFiles() throws IOException {
MockDirectoryWrapper dir = newDirectory();
dir.setPreventDoubleWrite(false);
+
+ LogMergePolicy mergePolicy = newLogMergePolicy(true, 10);
+ mergePolicy.setNoCFSRatio(1); // This test expects all of its segments to be in CFS
+
IndexWriter writer = new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
setMaxBufferedDocs(10).
- setMergePolicy(newLogMergePolicy(true, 10))
+ setMergePolicy(mergePolicy)
);
+
int i;
for(i=0;i<35;i++) {
addDoc(writer, i);
}
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false);
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false);
+ mergePolicy.setUseCompoundFile(false);
+ mergePolicy.setUseCompoundDocStore(false);
for(;i<45;i++) {
addDoc(writer, i);
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Dec 8 18:35:41 2010
@@ -2479,10 +2479,14 @@ public class TestIndexWriter extends Luc
public void testDeleteUnusedFiles() throws Exception {
for(int iter=0;iter<2;iter++) {
Directory dir = newDirectory();
+
+ LogMergePolicy mergePolicy = newLogMergePolicy(true);
+ mergePolicy.setNoCFSRatio(1); // This test expects all of its segments to be in CFS
+
IndexWriter w = new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
- setMergePolicy(newLogMergePolicy(true))
+ setMergePolicy(mergePolicy)
);
Document doc = new Document();
doc.add(newField("field", "go", Field.Store.NO, Field.Index.ANALYZED));
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java?rev=1043582&r1=1043581&r2=1043582&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java Wed Dec 8 18:35:41 2010
@@ -18,6 +18,7 @@
package org.apache.lucene.util;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
@@ -59,7 +60,7 @@ public class LuceneJUnitResultFormatter
private String systemError = null;
/** Buffer output until the end of the test */
- private StringBuilder sb;
+ private ByteArrayOutputStream sb; // use a BOS for our mostly ascii-output
private static final org.apache.lucene.store.Lock lock;
@@ -80,7 +81,6 @@ public class LuceneJUnitResultFormatter
/** Constructor for LuceneJUnitResultFormatter. */
public LuceneJUnitResultFormatter() {
- sb = new StringBuilder();
}
/**
@@ -116,13 +116,13 @@ public class LuceneJUnitResultFormatter
if (out == null) {
return; // Quick return - no output do nothing.
}
+ sb = new ByteArrayOutputStream(); // don't reuse, so its gc'ed
try {
LogManager.getLogManager().readConfiguration();
} catch (Exception e) {}
- sb.setLength(0);
- sb.append("Testsuite: ");
- sb.append(suite.getName());
- sb.append(StringUtils.LINE_SEP);
+ append("Testsuite: ");
+ append(suite.getName());
+ append(StringUtils.LINE_SEP);
}
/**
@@ -130,21 +130,21 @@ public class LuceneJUnitResultFormatter
* @param suite the test suite
*/
public synchronized void endTestSuite(JUnitTest suite) {
- sb.append("Tests run: ");
- sb.append(suite.runCount());
- sb.append(", Failures: ");
- sb.append(suite.failureCount());
- sb.append(", Errors: ");
- sb.append(suite.errorCount());
- sb.append(", Time elapsed: ");
- sb.append(numberFormat.format(suite.getRunTime() / ONE_SECOND));
- sb.append(" sec");
- sb.append(StringUtils.LINE_SEP);
- sb.append(StringUtils.LINE_SEP);
+ append("Tests run: ");
+ append(suite.runCount());
+ append(", Failures: ");
+ append(suite.failureCount());
+ append(", Errors: ");
+ append(suite.errorCount());
+ append(", Time elapsed: ");
+ append(numberFormat.format(suite.getRunTime() / ONE_SECOND));
+ append(" sec");
+ append(StringUtils.LINE_SEP);
+ append(StringUtils.LINE_SEP);
// append the err and output streams to the log
if (systemOutput != null && systemOutput.length() > 0) {
- sb.append("------------- Standard Output ---------------")
+ append("------------- Standard Output ---------------")
.append(StringUtils.LINE_SEP)
.append(systemOutput)
.append("------------- ---------------- ---------------")
@@ -152,7 +152,7 @@ public class LuceneJUnitResultFormatter
}
if (systemError != null && systemError.length() > 0) {
- sb.append("------------- Standard Error -----------------")
+ append("------------- Standard Error -----------------")
.append(StringUtils.LINE_SEP)
.append(systemError)
.append("------------- ---------------- ---------------")
@@ -163,7 +163,7 @@ public class LuceneJUnitResultFormatter
try {
lock.obtain(5000);
try {
- out.write(sb.toString().getBytes());
+ sb.writeTo(out);
out.flush();
} finally {
try {
@@ -252,14 +252,29 @@ public class LuceneJUnitResultFormatter
endTest(test);
}
- sb.append(formatTest(test) + type);
- sb.append(StringUtils.LINE_SEP);
- sb.append(error.getMessage());
- sb.append(StringUtils.LINE_SEP);
+ append(formatTest(test) + type);
+ append(StringUtils.LINE_SEP);
+ append(error.getMessage());
+ append(StringUtils.LINE_SEP);
String strace = JUnitTestRunner.getFilteredTrace(error);
- sb.append(strace);
- sb.append(StringUtils.LINE_SEP);
- sb.append(StringUtils.LINE_SEP);
+ append(strace);
+ append(StringUtils.LINE_SEP);
+ append(StringUtils.LINE_SEP);
+ }
+
+ public LuceneJUnitResultFormatter append(String s) {
+ if (s == null)
+ s = "(null)";
+ try {
+ sb.write(s.getBytes()); // intentionally use default charset, its a console.
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ public LuceneJUnitResultFormatter append(long l) {
+ return append(Long.toString(l));
}
}