You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/06/04 20:42:15 UTC
svn commit: r951521 - in /lucene/dev/trunk/lucene: ./
src/java/org/apache/lucene/index/ src/test/org/apache/lucene/index/
Author: mikemccand
Date: Fri Jun 4 18:42:15 2010
New Revision: 951521
URL: http://svn.apache.org/viewvc?rev=951521&view=rev
Log:
LUCENE-2486: incRef the doc store files so we can load them when opening the reader for warming
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=951521&r1=951520&r2=951521&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Jun 4 18:42:15 2010
@@ -472,6 +472,10 @@ Bug fixes
that warming is free to do whatever it needs to. (Earwin Burrfoot
via Mike McCandless)
+* LUCENE-2486: Fixed intermittent FileNotFoundException on doc store
+ files when a mergedSegmentWarmer is set on IndexWriter. (Mike
+ McCandless)
+
New features
* LUCENE-2128: Parallelized fetching document frequencies during weight
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=951521&r1=951520&r2=951521&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Jun 4 18:42:15 2010
@@ -465,7 +465,7 @@ final class IndexFileDeleter {
}
}
- void incRef(List<String> files) throws IOException {
+ void incRef(Collection<String> files) throws IOException {
for(final String file : files) {
incRef(file);
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=951521&r1=951520&r2=951521&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java Fri Jun 4 18:42:15 2010
@@ -3537,25 +3537,7 @@ public class IndexWriter implements Clos
commitMergedDeletes(merge, mergedReader);
docWriter.remapDeletes(segmentInfos, merger.getDocMaps(), merger.getDelCounts(), merge, mergedDocCount);
- // Simple optimization: if the doc store we are using
- // has been closed and is in now compound format (but
- // wasn't when we started), then we will switch to the
- // compound format as well:
- final String mergeDocStoreSegment = merge.info.getDocStoreSegment();
- if (mergeDocStoreSegment != null && !merge.info.getDocStoreIsCompoundFile()) {
- final int size = segmentInfos.size();
- for(int i=0;i<size;i++) {
- final SegmentInfo info = segmentInfos.info(i);
- final String docStoreSegment = info.getDocStoreSegment();
- if (docStoreSegment != null &&
- docStoreSegment.equals(mergeDocStoreSegment) &&
- info.getDocStoreIsCompoundFile()) {
- merge.info.setDocStoreIsCompoundFile(true);
- break;
- }
- }
- }
-
+ setMergeDocStoreIsCompoundFile(merge);
merge.info.setHasProx(merger.hasProx());
segmentInfos.subList(start, start + merge.segments.size()).clear();
@@ -3903,6 +3885,11 @@ public class IndexWriter implements Clos
if (merge.increfDone)
decrefMergeSegments(merge);
+ if (merge.mergeFiles != null) {
+ deleter.decRef(merge.mergeFiles);
+ merge.mergeFiles = null;
+ }
+
// It's possible we are called twice, eg if there was an
// exception inside mergeInit
if (merge.registerDone) {
@@ -3917,6 +3904,23 @@ public class IndexWriter implements Clos
runningMerges.remove(merge);
}
+ private synchronized void setMergeDocStoreIsCompoundFile(MergePolicy.OneMerge merge) {
+ final String mergeDocStoreSegment = merge.info.getDocStoreSegment();
+ if (mergeDocStoreSegment != null && !merge.info.getDocStoreIsCompoundFile()) {
+ final int size = segmentInfos.size();
+ for(int i=0;i<size;i++) {
+ final SegmentInfo info = segmentInfos.info(i);
+ final String docStoreSegment = info.getDocStoreSegment();
+ if (docStoreSegment != null &&
+ docStoreSegment.equals(mergeDocStoreSegment) &&
+ info.getDocStoreIsCompoundFile()) {
+ merge.info.setDocStoreIsCompoundFile(true);
+ break;
+ }
+ }
+ }
+ }
+
/** Does the actual (time-consuming) work of the merge,
* but without holding synchronized lock on IndexWriter
* instance */
@@ -4037,6 +4041,17 @@ public class IndexWriter implements Clos
final int termsIndexDivisor;
final boolean loadDocStores;
+ synchronized(this) {
+ // If the doc store we are using has been closed and
+ // is in now compound format (but wasn't when we
+ // started), then we will switch to the compound
+ // format as well:
+ setMergeDocStoreIsCompoundFile(merge);
+ assert merge.mergeFiles == null;
+ merge.mergeFiles = merge.info.files();
+ deleter.incRef(merge.mergeFiles);
+ }
+
if (poolReaders && mergedSegmentWarmer != null) {
// Load terms index & doc stores so the segment
// warmer can run searches, load documents/term
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=951521&r1=951520&r2=951521&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java Fri Jun 4 18:42:15 2010
@@ -76,6 +76,7 @@ public abstract class MergePolicy implem
int maxNumSegmentsOptimize; // used by IndexWriter
SegmentReader[] readers; // used by IndexWriter
SegmentReader[] readersClone; // used by IndexWriter
+ List<String> mergeFiles; // used by IndexWriter
final SegmentInfos segments;
final boolean useCompoundFile;
boolean aborted;
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=951521&r1=951520&r2=951521&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Fri Jun 4 18:42:15 2010
@@ -542,7 +542,7 @@ public class TestIndexWriterReader exten
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < 100*_TestUtil.getRandomMultiplier(); i++) {
writer.addDocument(createDocument(i, "test", 4));
}
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();