You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/11/20 11:34:41 UTC
svn commit: r1037159 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/ lucene/backwards/src/test/org/apache/lucene/analysis/
lucene/backwards/src/test/org/apache/lucene/document/
lucene/backwards/src/test/org/apache/lucene/index/ lu...
Author: mikemccand
Date: Sat Nov 20 10:34:39 2010
New Revision: 1037159
URL: http://svn.apache.org/viewvc?rev=1037159&view=rev
Log:
LUCENE-2762: fix IW to not hold open but deleted files when CFS is true
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
lucene/dev/branches/branch_3x/lucene/build.xml (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ca/CatalanAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishMinimalStemmer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishPossessiveFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/eu/BasqueAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hy/ArmenianAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/util/StemmerUtil.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/tartarus/snowball/TestApp.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/stempel/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/lib/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/BaseCharFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/LetterTokenizer.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/StandardFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/UAX29Tokenizer.jflex (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/StringHelper.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/LuceneResourcesWikiPage.html (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/LuceneResourcesWikiPageURLs.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/email.addresses.from.random.text.with.email.addresses.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/random.text.with.email.addresses.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/random.text.with.urls.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/urls.from.random.text.with.urls.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/collation/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/example/ (props changed)
lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar (props changed)
lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar (props changed)
lucene/dev/branches/branch_3x/solr/src/ (props changed)
lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common/ (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/ShingleFilterFactory.java (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/SynonymFilter.java (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/SynonymMap.java (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/WordDelimiterIterator.java (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/response/ (props changed)
lucene/dev/branches/branch_3x/solr/src/maven/solr-core-pom.xml.template (props changed)
lucene/dev/branches/branch_3x/solr/src/maven/solr-solrj-pom.xml.template (props changed)
lucene/dev/branches/branch_3x/solr/src/solrj/org/ (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilter.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestSynonymFilter.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestTrimFilter.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/client/ (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/request/SimpleFacetsTest.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/update/AutoCommitTest.java (props changed)
lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/ (props changed)
lucene/dev/branches/branch_3x/solr/src/webapp/web/admin/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Sat Nov 20 10:34:39 2010
@@ -112,6 +112,13 @@ Changes in runtime behavior
* LUCENE-2753: IndexReader and DirectoryReader .listCommits() now return a List,
guaranteeing the commits are sorted from oldest to latest. (Shai Erera)
+* LUCENE-2762: Fixed bug in IndexWriter causing it to hold open file
+ handles against deleted files when compound-file was enabled (the
+ default) and readers are pooled. As a result of this the peak
+ worst-case free disk space required during optimize is now 3X the
+ index size, when compound file is enabled (else 2X). (Mike
+ McCandless)
+
API Changes
* LUCENE-2076: Rename FSDirectory.getFile -> getDirectory. (George
@@ -122,7 +129,7 @@ API Changes
custom Similarity can alter how norms are encoded, though they must
still be encoded as a single byte (Johan Kindgren via Mike
McCandless)
-
+
* LUCENE-2103: NoLockFactory should have a private constructor;
until Lucene 4.0 the default one will be deprecated.
(Shai Erera via Uwe Schindler)
@@ -322,6 +329,13 @@ Bug fixes
not the number that have norms enabled, on the "test: field
norms..." output. (Mark Kristensson via Mike McCandless)
+* LUCENE-2762: Fixed bug in IndexWriter causing it to hold open file
+ handles against deleted files when compound-file was enabled (the
+ default) and readers are pooled. As a result of this the peak
+ worst-case free disk space required during optimize is now 3X the
+ index size, when compound file is enabled (else 2X). (Mike
+ McCandless)
+
New features
* LUCENE-2128: Parallelized fetching document frequencies during weight
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Sat Nov 20 10:34:39 2010
@@ -1855,6 +1855,9 @@ public class IndexWriter implements Clos
}
boolean useCompoundDocStore = false;
+ if (infoStream != null) {
+ message("closeDocStores segment=" + docWriter.getDocStoreSegment());
+ }
String docStoreSegment;
@@ -2310,11 +2313,12 @@ public class IndexWriter implements Clos
* calling optimize. </p>
*
* <p>Note that optimize requires 2X the index size free
- * space in your Directory. For example, if your index
- * size is 10 MB then you need 20 MB free for optimize to
- * complete. Also, it's best to call {@link #commit()}
- * after the optimize completes to allow IndexWriter to
- * free up disk space.</p>
+ * space in your Directory (3X if you're using compound
+ * file format). For example, if your index size is 10 MB
+ * then you need 20 MB free for optimize to complete (30
+ * MB if you're using compound file format). Also,
+ * it's best to call {@link #commit()} after the optimize
+ * completes to allow IndexWriter to free up disk space.</p>
*
* <p>If some but not all readers re-open while an
* optimize is underway, this will cause > 2X temporary
@@ -3638,8 +3642,6 @@ public class IndexWriter implements Clos
if (merge.isAborted()) {
if (infoStream != null)
message("commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted");
-
- deleter.refresh(merge.info.name);
return false;
}
@@ -3648,13 +3650,20 @@ public class IndexWriter implements Clos
commitMergedDeletes(merge, mergedReader);
docWriter.remapDeletes(segmentInfos, merger.getDocMaps(), merger.getDelCounts(), merge, mergedDocCount);
+ // If the doc store we are using has been closed and
+ // is in now compound format (but wasn't when we
+ // started), then we will switch to the compound
+ // format as well:
setMergeDocStoreIsCompoundFile(merge);
+
merge.info.setHasProx(merger.hasProx());
segmentInfos.subList(start, start + merge.segments.size()).clear();
assert !segmentInfos.contains(merge.info);
segmentInfos.add(start, merge.info);
+ closeMergeReaders(merge, false);
+
// Must note the change to segmentInfos so any commits
// in-flight don't lose it:
checkpoint();
@@ -3671,11 +3680,6 @@ public class IndexWriter implements Clos
return true;
}
- private synchronized void decrefMergeSegments(MergePolicy.OneMerge merge) throws IOException {
- assert merge.increfDone;
- merge.increfDone = false;
- }
-
final private void handleMergeException(Throwable t, MergePolicy.OneMerge merge) throws IOException {
if (infoStream != null) {
@@ -3956,8 +3960,6 @@ public class IndexWriter implements Clos
updatePendingMerges(1, false);
}
- merge.increfDone = true;
-
merge.mergeDocStores = mergeDocStores;
// Bind a new segment name here so even with
@@ -4011,14 +4013,6 @@ public class IndexWriter implements Clos
// on merges to finish.
notifyAll();
- if (merge.increfDone)
- decrefMergeSegments(merge);
-
- if (merge.mergeFiles != null) {
- deleter.decRef(merge.mergeFiles);
- merge.mergeFiles = null;
- }
-
// It's possible we are called twice, eg if there was an
// exception inside mergeInit
if (merge.registerDone) {
@@ -4048,7 +4042,49 @@ public class IndexWriter implements Clos
}
}
}
- }
+ }
+
+ private final synchronized void closeMergeReaders(MergePolicy.OneMerge merge, boolean suppressExceptions) throws IOException {
+ final int numSegments = merge.segments.size();
+ if (suppressExceptions) {
+ // Suppress any new exceptions so we throw the
+ // original cause
+ for (int i=0;i<numSegments;i++) {
+ if (merge.readers[i] != null) {
+ try {
+ readerPool.release(merge.readers[i], false);
+ } catch (Throwable t) {
+ }
+ merge.readers[i] = null;
+ }
+
+ if (merge.readersClone[i] != null) {
+ try {
+ merge.readersClone[i].close();
+ } catch (Throwable t) {
+ }
+ // This was a private clone and we had the
+ // only reference
+ assert merge.readersClone[i].getRefCount() == 0: "refCount should be 0 but is " + merge.readersClone[i].getRefCount();
+ merge.readersClone[i] = null;
+ }
+ }
+ } else {
+ for (int i=0;i<numSegments;i++) {
+ if (merge.readers[i] != null) {
+ readerPool.release(merge.readers[i], true);
+ merge.readers[i] = null;
+ }
+
+ if (merge.readersClone[i] != null) {
+ merge.readersClone[i].close();
+ // This was a private clone and we had the only reference
+ assert merge.readersClone[i].getRefCount() == 0;
+ merge.readersClone[i] = null;
+ }
+ }
+ }
+ }
/** Does the actual (time-consuming) work of the merge,
* but without holding synchronized lock on IndexWriter
@@ -4077,8 +4113,13 @@ public class IndexWriter implements Clos
boolean mergeDocStores = false;
- final Set<String> dss = new HashSet<String>();
-
+ final String currentDocStoreSegment;
+ synchronized(this) {
+ currentDocStoreSegment = docWriter.getDocStoreSegment();
+ }
+
+ boolean currentDSSMerged = false;
+
// This is try/finally to make sure merger's readers are
// closed:
boolean success = false;
@@ -4086,7 +4127,6 @@ public class IndexWriter implements Clos
int totDocCount = 0;
for (int i = 0; i < numSegments; i++) {
-
final SegmentInfo info = sourceSegments.info(i);
// Hold onto the "live" reader; we will use this to
@@ -4105,8 +4145,8 @@ public class IndexWriter implements Clos
mergeDocStores = true;
}
- if (info.getDocStoreOffset() != -1) {
- dss.add(info.getDocStoreSegment());
+ if (info.getDocStoreOffset() != -1 && currentDocStoreSegment != null) {
+ currentDSSMerged |= currentDocStoreSegment.equals(info.getDocStoreSegment());
}
totDocCount += clone.numDocs();
@@ -4132,9 +4172,10 @@ public class IndexWriter implements Clos
// readers will attempt to open an IndexInput
// on files that have still-open IndexOutputs
// against them:
- if (dss.contains(docWriter.getDocStoreSegment())) {
- if (infoStream != null)
+ if (currentDSSMerged) {
+ if (infoStream != null) {
message("now flush at mergeMiddle");
+ }
doFlush(true, false);
updatePendingMerges(1, false);
}
@@ -4145,9 +4186,7 @@ public class IndexWriter implements Clos
}
// Clear DSS
- synchronized(this) {
- merge.info.setDocStore(-1, null, false);
- }
+ merge.info.setDocStore(-1, null, false);
}
// This is where all the work happens:
@@ -4155,26 +4194,65 @@ public class IndexWriter implements Clos
assert mergedDocCount == totDocCount;
- // TODO: in the non-realtime case, we may want to only
- // keep deletes (it's costly to open entire reader
- // when we just need deletes)
+ if (merge.useCompoundFile) {
- final int termsIndexDivisor;
- final boolean loadDocStores;
+ success = false;
+ final String compoundFileName = IndexFileNames.segmentFileName(mergedName, IndexFileNames.COMPOUND_FILE_EXTENSION);
- synchronized(this) {
- // If the doc store we are using has been closed and
- // is in now compound format (but wasn't when we
- // started), then we will switch to the compound
- // format as well:
- setMergeDocStoreIsCompoundFile(merge);
- assert merge.mergeFiles == null;
- merge.mergeFiles = merge.info.files();
- deleter.incRef(merge.mergeFiles);
+ try {
+ if (infoStream != null) {
+ message("create compound file " + compoundFileName);
+ }
+ merger.createCompoundFile(compoundFileName);
+ success = true;
+ } catch (IOException ioe) {
+ synchronized(this) {
+ if (merge.isAborted()) {
+ // This can happen if rollback or close(false)
+ // is called -- fall through to logic below to
+ // remove the partially created CFS:
+ } else {
+ handleMergeException(ioe, merge);
+ }
+ }
+ } catch (Throwable t) {
+ handleMergeException(t, merge);
+ } finally {
+ if (!success) {
+ if (infoStream != null) {
+ message("hit exception creating compound file during merge");
+ }
+
+ synchronized(this) {
+ deleter.deleteFile(compoundFileName);
+ deleter.deleteNewFiles(merger.getMergedFiles());
+ }
+ }
+ }
+
+ success = false;
+
+ synchronized(this) {
+
+ // delete new non cfs files directly: they were never
+ // registered with IFD
+ deleter.deleteNewFiles(merger.getMergedFiles());
+
+ if (merge.isAborted()) {
+ if (infoStream != null) {
+ message("abort merge after building CFS");
+ }
+ deleter.deleteFile(compoundFileName);
+ return 0;
+ }
+ }
+
+ merge.info.setUseCompoundFile(true);
}
- final String currentDocStoreSegment = docWriter.getDocStoreSegment();
-
+ final int termsIndexDivisor;
+ final boolean loadDocStores;
+
// if the merged segment warmer was not installed when
// this merge was started, causing us to not force
// the docStores to close, we can't warm it now
@@ -4191,116 +4269,32 @@ public class IndexWriter implements Clos
loadDocStores = false;
}
+ // TODO: in the non-realtime case, we may want to only
+ // keep deletes (it's costly to open entire reader
+ // when we just need deletes)
+
final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, BufferedIndexInput.BUFFER_SIZE, termsIndexDivisor);
try {
if (poolReaders && mergedSegmentWarmer != null) {
mergedSegmentWarmer.warm(mergedReader);
}
- if (!commitMerge(merge, merger, mergedDocCount, mergedReader))
+
+ if (!commitMerge(merge, merger, mergedDocCount, mergedReader)) {
// commitMerge will return false if this merge was aborted
return 0;
+ }
} finally {
synchronized(this) {
readerPool.release(mergedReader);
}
}
-
success = true;
- } finally {
- synchronized(this) {
- if (!success) {
- // Suppress any new exceptions so we throw the
- // original cause
- for (int i=0;i<numSegments;i++) {
- if (merge.readers[i] != null) {
- try {
- readerPool.release(merge.readers[i], false);
- } catch (Throwable t) {
- }
- }
-
- if (merge.readersClone[i] != null) {
- try {
- merge.readersClone[i].close();
- } catch (Throwable t) {
- }
- // This was a private clone and we had the only reference
- assert merge.readersClone[i].getRefCount() == 0;
- }
- }
- } else {
- for (int i=0;i<numSegments;i++) {
- if (merge.readers[i] != null) {
- readerPool.release(merge.readers[i], true);
- }
-
- if (merge.readersClone[i] != null) {
- merge.readersClone[i].close();
- // This was a private clone and we had the only reference
- assert merge.readersClone[i].getRefCount() == 0;
- }
- }
- }
- }
- }
-
- // Must checkpoint before decrefing so any newly
- // referenced files in the new merge.info are incref'd
- // first:
- synchronized(this) {
- deleter.checkpoint(segmentInfos, false);
- }
- decrefMergeSegments(merge);
-
- if (merge.useCompoundFile) {
-
- success = false;
- final String compoundFileName = IndexFileNames.segmentFileName(mergedName, IndexFileNames.COMPOUND_FILE_EXTENSION);
-
- try {
- merger.createCompoundFile(compoundFileName);
- success = true;
- } catch (IOException ioe) {
- synchronized(this) {
- if (merge.isAborted()) {
- // This can happen if rollback or close(false)
- // is called -- fall through to logic below to
- // remove the partially created CFS:
- success = true;
- } else
- handleMergeException(ioe, merge);
- }
- } catch (Throwable t) {
- handleMergeException(t, merge);
- } finally {
- if (!success) {
- if (infoStream != null)
- message("hit exception creating compound file during merge");
- synchronized(this) {
- deleter.deleteFile(compoundFileName);
- }
- }
- }
- if (merge.isAborted()) {
- if (infoStream != null)
- message("abort merge after building CFS");
- synchronized(this) {
- deleter.deleteFile(compoundFileName);
- }
- return 0;
- }
-
- synchronized(this) {
- if (segmentInfos.indexOf(merge.info) == -1 || merge.isAborted()) {
- // Our segment (committed in non-compound
- // format) got merged away while we were
- // building the compound format.
- deleter.deleteFile(compoundFileName);
- } else {
- merge.info.setUseCompoundFile(true);
- checkpoint();
- }
+ } finally {
+ // Readers are already closed in commitMerge if we didn't hit
+ // an exc:
+ if (!success) {
+ closeMergeReaders(merge, true);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java Sat Nov 20 10:34:39 2010
@@ -69,14 +69,12 @@ public abstract class MergePolicy implem
SegmentInfo info; // used by IndexWriter
boolean mergeDocStores; // used by IndexWriter
boolean optimize; // used by IndexWriter
- boolean increfDone; // used by IndexWriter
boolean registerDone; // used by IndexWriter
long mergeGen; // used by IndexWriter
boolean isExternal; // used by IndexWriter
int maxNumSegmentsOptimize; // used by IndexWriter
SegmentReader[] readers; // used by IndexWriter
SegmentReader[] readersClone; // used by IndexWriter
- List<String> mergeFiles; // used by IndexWriter
public final SegmentInfos segments;
public final boolean useCompoundFile;
boolean aborted;
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Sat Nov 20 10:34:39 2010
@@ -20,6 +20,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
import java.util.List;
@@ -155,13 +157,8 @@ final class SegmentMerger {
return mergedDocs;
}
- final List<String> createCompoundFile(String fileName)
- throws IOException {
- CompoundFileWriter cfsWriter =
- new CompoundFileWriter(directory, fileName, checkAbort);
-
- List<String> files =
- new ArrayList<String>(IndexFileNames.COMPOUND_EXTENSIONS.length + 1);
+ final Collection<String> getMergedFiles() throws IOException {
+ Set<String> fileSet = new HashSet<String>();
// Basic files
for (String ext : IndexFileNames.COMPOUND_EXTENSIONS) {
@@ -170,7 +167,7 @@ final class SegmentMerger {
if (mergeDocStores || (!ext.equals(IndexFileNames.FIELDS_EXTENSION) &&
!ext.equals(IndexFileNames.FIELDS_INDEX_EXTENSION)))
- files.add(IndexFileNames.segmentFileName(segment, ext));
+ fileSet.add(IndexFileNames.segmentFileName(segment, ext));
}
// Fieldable norm files
@@ -178,7 +175,7 @@ final class SegmentMerger {
for (int i = 0; i < numFIs; i++) {
FieldInfo fi = fieldInfos.fieldInfo(i);
if (fi.isIndexed && !fi.omitNorms) {
- files.add(IndexFileNames.segmentFileName(segment, IndexFileNames.NORMS_EXTENSION));
+ fileSet.add(IndexFileNames.segmentFileName(segment, IndexFileNames.NORMS_EXTENSION));
break;
}
}
@@ -186,11 +183,19 @@ final class SegmentMerger {
// Vector files
if (fieldInfos.hasVectors() && mergeDocStores) {
for (String ext : IndexFileNames.VECTOR_EXTENSIONS) {
- files.add(IndexFileNames.segmentFileName(segment, ext));
+ fileSet.add(IndexFileNames.segmentFileName(segment, ext));
}
}
+ return fileSet;
+ }
+
+ final Collection<String> createCompoundFile(String fileName)
+ throws IOException {
+
// Now merge all added files
+ Collection<String> files = getMergedFiles();
+ CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
for (String file : files) {
cfsWriter.addFile(file);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Sat Nov 20 10:34:39 2010
@@ -844,7 +844,7 @@ public class TestAddIndexes extends Luce
}
c.launchThreads(-1);
- Thread.sleep(500);
+ Thread.sleep(_TestUtil.nextInt(random, 10, 500));
// Close w/o first stopping/joining the threads
if (VERBOSE) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java Sat Nov 20 10:34:39 2010
@@ -24,8 +24,7 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
+import java.util.Collection;
import junit.framework.TestSuite;
import junit.textui.TestRunner;
@@ -197,7 +196,7 @@ public class TestDoc extends LuceneTestC
r2.close();
if (useCompoundFile) {
- List<String> filesToDelete = merger.createCompoundFile(merged + ".cfs");
+ Collection<String> filesToDelete = merger.createCompoundFile(merged + ".cfs");
for (final String fileToDelete : filesToDelete)
si1.dir.deleteFile(fileToDelete);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Nov 20 10:34:39 2010
@@ -319,29 +319,46 @@ public class TestIndexWriter extends Luc
* required.
*/
public void testOptimizeTempSpaceUsage() throws IOException {
-
+
MockDirectoryWrapper dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(10));
+ if (VERBOSE) {
+ System.out.println("TEST: config1=" + writer.getConfig());
+ }
for(int j=0;j<500;j++) {
addDocWithIndex(writer, j);
}
+ final int termIndexInterval = writer.getConfig().getTermIndexInterval();
+ // force one extra segment w/ different doc store so
+ // we see the doc stores get merged
+ writer.commit();
+ addDocWithIndex(writer, 500);
writer.close();
+ if (VERBOSE) {
+ System.out.println("TEST: start disk usage");
+ }
long startDiskUsage = 0;
String[] files = dir.listAll();
for(int i=0;i<files.length;i++) {
startDiskUsage += dir.fileLength(files[i]);
+ if (VERBOSE) {
+ System.out.println(files[i] + ": " + dir.fileLength(files[i]));
+ }
}
dir.resetMaxUsedSizeInBytes();
dir.setTrackDiskUsage(true);
- writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND));
+ // Import to use same term index interval else a
+ // smaller one here could increase the disk usage and
+ // cause a false failure:
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND).setTermIndexInterval(termIndexInterval));
+ writer.setInfoStream(VERBOSE ? System.out : null);
writer.optimize();
writer.close();
long maxDiskUsage = dir.getMaxUsedSizeInBytes();
-
- assertTrue("optimized used too much temporary space: starting usage was " + startDiskUsage + " bytes; max temp usage was " + maxDiskUsage + " but should have been " + (3*startDiskUsage) + " (= 3X starting usage)",
- maxDiskUsage <= 3*startDiskUsage);
+ assertTrue("optimize used too much temporary space: starting usage was " + startDiskUsage + " bytes; max temp usage was " + maxDiskUsage + " but should have been " + (4*startDiskUsage) + " (= 4X starting usage)",
+ maxDiskUsage <= 4*startDiskUsage);
dir.close();
}
@@ -563,15 +580,15 @@ public class TestIndexWriter extends Luc
long endDiskUsage = dir.getMaxUsedSizeInBytes();
// Ending index is 50X as large as starting index; due
- // to 2X disk usage normally we allow 100X max
+ // to 3X disk usage normally we allow 150X max
// transient usage. If something is wrong w/ deleter
// and it doesn't delete intermediate segments then it
- // will exceed this 100X:
+ // will exceed this 150X:
// System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
- assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage,
- midDiskUsage < 100*startDiskUsage);
- assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage,
- endDiskUsage < 100*startDiskUsage);
+ assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage*150),
+ midDiskUsage < 150*startDiskUsage);
+ assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage*150),
+ endDiskUsage < 150*startDiskUsage);
dir.close();
}
@@ -594,6 +611,10 @@ public class TestIndexWriter extends Luc
writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND));
writer.optimize();
+ if (VERBOSE) {
+ writer.setInfoStream(System.out);
+ }
+
// Open a reader before closing (commiting) the writer:
IndexReader reader = IndexReader.open(dir, true);
@@ -613,9 +634,19 @@ public class TestIndexWriter extends Luc
assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
reader.close();
+ if (VERBOSE) {
+ System.out.println("TEST: do real optimize");
+ }
writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND));
+ if (VERBOSE) {
+ writer.setInfoStream(System.out);
+ }
writer.optimize();
writer.close();
+
+ if (VERBOSE) {
+ System.out.println("TEST: writer closed");
+ }
assertNoUnreferencedFiles(dir, "aborted writer after optimize");
// Open a reader after aborting writer:
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java Sat Nov 20 10:34:39 2010
@@ -24,7 +24,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.LuceneTestCase;
@@ -219,7 +218,7 @@ public class TestIndexWriterMergePolicy
}
private void checkInvariants(IndexWriter writer) throws IOException {
- _TestUtil.syncConcurrentMerges(writer);
+ writer.waitForMerges();
int maxBufferedDocs = writer.getConfig().getMaxBufferedDocs();
int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor();
int maxMergeDocs = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMaxMergeDocs();
@@ -261,7 +260,7 @@ public class TestIndexWriterMergePolicy
segmentCfsCount++;
}
}
- assertEquals(segmentCount, segmentCfsCount);
+ assertEquals("index=" + writer.segString(), segmentCount, segmentCfsCount);
}
/*
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Sat Nov 20 10:34:39 2010
@@ -61,9 +61,17 @@ public class TestIndexWriterReader exten
boolean optimize = true;
Directory dir1 = newDirectory();
- IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
- // test relies on no merges happening below:
- ((LogMergePolicy) writer.getMergePolicy()).setMergeFactor(10);
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
+ if (iwc.getMaxBufferedDocs() < 20) {
+ iwc.setMaxBufferedDocs(20);
+ }
+ // no merging
+ if (random.nextBoolean()) {
+ iwc.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
+ } else {
+ iwc.setMergePolicy(NoMergePolicy.COMPOUND_FILES);
+ }
+ IndexWriter writer = new IndexWriter(dir1, iwc);
// create the index
createIndexNoClose(!optimize, "index1", writer);
@@ -124,9 +132,17 @@ public class TestIndexWriterReader exten
boolean optimize = false;
Directory dir1 = newDirectory();
- IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
- // test relies on no merges happening below:
- ((LogMergePolicy) writer.getMergePolicy()).setMergeFactor(10);
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
+ if (iwc.getMaxBufferedDocs() < 20) {
+ iwc.setMaxBufferedDocs(20);
+ }
+ // no merging
+ if (random.nextBoolean()) {
+ iwc.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
+ } else {
+ iwc.setMergePolicy(NoMergePolicy.COMPOUND_FILES);
+ }
+ IndexWriter writer = new IndexWriter(dir1, iwc);
writer.setInfoStream(infoStream);
// create the index
@@ -260,6 +276,13 @@ public class TestIndexWriterReader exten
Directory mainDir = newDirectory();
IndexWriter mainWriter = new IndexWriter(mainDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
+
+ // try to keep open file count down:
+ LogMergePolicy lmp = (LogMergePolicy) mainWriter.getMergePolicy();
+ if (lmp.getMergeFactor() > 5) {
+ lmp.setMergeFactor(5);
+ }
+
mainWriter.setInfoStream(infoStream);
AddDirectoriesThreads addDirThreads = new AddDirectoriesThreads(numIter, mainWriter);
addDirThreads.launchThreads(numDirs);
@@ -615,7 +638,7 @@ public class TestIndexWriterReader exten
// Stress test reopen during addIndexes
public void testDuringAddIndexes() throws Exception {
- Directory dir1 = newDirectory();
+ MockDirectoryWrapper dir1 = newDirectory();
final IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
writer.setInfoStream(infoStream);
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);
@@ -684,10 +707,12 @@ public class TestIndexWriterReader exten
assertTrue(count >= lastCount);
assertEquals(0, excs.size());
+ r.close();
+ assertEquals(0, dir1.getOpenDeletedFiles().size());
+
writer.close();
_TestUtil.checkIndex(dir1);
- r.close();
dir1.close();
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java?rev=1037159&r1=1037158&r2=1037159&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java Sat Nov 20 10:34:39 2010
@@ -111,10 +111,8 @@ public class TestThreadedOptimize extend
final int expectedDocCount = (int) ((1+iter)*(200+8*NUM_ITER2*(NUM_THREADS/2.0)*(1+NUM_THREADS)));
- // System.out.println("TEST: now index=" + writer.segString());
-
- assertEquals("index=" + writer.segString() + " numDocs" + writer.numDocs() + " maxDoc=" + writer.maxDoc() + " config=" + writer.getConfig(), expectedDocCount, writer.numDocs());
- assertEquals("index=" + writer.segString() + " numDocs" + writer.numDocs() + " maxDoc=" + writer.maxDoc() + " config=" + writer.getConfig(), expectedDocCount, writer.maxDoc());
+ assertEquals("index=" + writer.segString() + " numDocs=" + writer.numDocs() + " maxDoc=" + writer.maxDoc() + " config=" + writer.getConfig(), expectedDocCount, writer.numDocs());
+ assertEquals("index=" + writer.segString() + " numDocs=" + writer.numDocs() + " maxDoc=" + writer.maxDoc() + " config=" + writer.getConfig(), expectedDocCount, writer.maxDoc());
writer.close();
writer = new IndexWriter(directory, newIndexWriterConfig(