You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2010/11/14 12:15:31 UTC
svn commit: r1034980 - in /lucene/java/branches/lucene_2_9: CHANGES.txt
src/java/org/apache/lucene/index/DocumentsWriter.java
src/java/org/apache/lucene/index/IndexWriter.java
src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
Author: mikemccand
Date: Sun Nov 14 11:15:30 2010
New Revision: 1034980
URL: http://svn.apache.org/viewvc?rev=1034980&view=rev
Log:
LUCENE-2759: fix 2 cases where we could open doc stores for read when they were still open for write
Modified:
lucene/java/branches/lucene_2_9/CHANGES.txt
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
Modified: lucene/java/branches/lucene_2_9/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/CHANGES.txt?rev=1034980&r1=1034979&r2=1034980&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/CHANGES.txt Sun Nov 14 11:15:30 2010
@@ -81,6 +81,10 @@ Bug fixes
not the number that have norms enabled, on the "test: field
norms..." output. (Mark Kristensson via Mike McCandless)
+* LUCENE-2759: Fixed two near-real-time cases where doc store files
+ may be opened for read even though they are still open for write.
+ (Mike McCandless)
+
Optimizations
* LUCENE-2556: Improve memory usage after cloning TermAttribute.
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1034980&r1=1034979&r2=1034980&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java Sun Nov 14 11:15:30 2010
@@ -538,6 +538,9 @@ final class DocumentsWriter {
} finally {
aborting = false;
notifyAll();
+ if (infoStream != null) {
+ message("docWriter: done abort; abortedFiles=" + abortedFiles);
+ }
}
}
@@ -661,6 +664,10 @@ final class DocumentsWriter {
return flushState.numDocs;
}
+ Collection getFlushedFiles() {
+ return flushState.flushedFiles;
+ }
+
/** Build compound file for the segment we just flushed */
void createCompoundFile(String segment) throws IOException {
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java?rev=1034980&r1=1034979&r2=1034980&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java Sun Nov 14 11:15:30 2010
@@ -477,15 +477,16 @@ public class IndexWriter {
// this method is called:
poolReaders = true;
- flush(true, true, false);
-
// Prevent segmentInfos from changing while opening the
// reader; in theory we could do similar retry logic,
// just like we do when loading segments_N
+ IndexReader r;
synchronized(this) {
- applyDeletes();
- return new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor);
+ flush(false, true, true);
+ r = new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor);
}
+ maybeMerge();
+ return r;
}
/** Holds shared SegmentReader instances. IndexWriter uses
@@ -2299,6 +2300,10 @@ public class IndexWriter {
*/
private synchronized boolean flushDocStores() throws IOException {
+ if (infoStream != null) {
+ message("flushDocStores segment=" + docWriter.getDocStoreSegment());
+ }
+
boolean useCompoundDocStore = false;
String docStoreSegment;
@@ -2313,6 +2318,10 @@ public class IndexWriter {
}
}
+ if (infoStream != null) {
+ message("flushDocStores files=" + docWriter.closedFiles());
+ }
+
useCompoundDocStore = mergePolicy.useCompoundDocStore(segmentInfos);
if (useCompoundDocStore && docStoreSegment != null && docWriter.closedFiles().size() != 0) {
@@ -4373,6 +4382,9 @@ public class IndexWriter {
try {
flushedDocCount = docWriter.flush(flushDocStores);
+ if (infoStream != null) {
+ message("flushedFiles=" + docWriter.getFlushedFiles());
+ }
success = true;
} finally {
if (!success) {
@@ -4858,6 +4870,13 @@ public class IndexWriter {
}
}
+ // if a mergedSegmentWarmer is installed, we must merge
+ // the doc stores because we will open a full
+ // SegmentReader on the merged segment:
+ if (!mergeDocStores && mergedSegmentWarmer != null && currentDocStoreSegment != null && lastDocStoreSegment != null && lastDocStoreSegment.equals(currentDocStoreSegment)) {
+ mergeDocStores = true;
+ }
+
final int docStoreOffset;
final String docStoreSegment;
final boolean docStoreIsCompoundFile;
@@ -5132,7 +5151,14 @@ public class IndexWriter {
deleter.incRef(merge.mergeFiles);
}
- if (poolReaders && mergedSegmentWarmer != null) {
+ final String currentDocStoreSegment = docWriter.getDocStoreSegment();
+
+ // if the merged segment warmer was not installed when
+ // this merge was started, causing us to not force
+ // the docStores to close, we can't warm it now
+ final boolean canWarm = merge.info.getDocStoreSegment() == null || currentDocStoreSegment == null || !merge.info.getDocStoreSegment().equals(currentDocStoreSegment);
+
+ if (poolReaders && mergedSegmentWarmer != null && canWarm) {
// Load terms index & doc stores so the segment
// warmer can run searches, load documents/term
// vectors
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=1034980&r1=1034979&r2=1034980&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Sun Nov 14 11:15:30 2010
@@ -53,6 +53,13 @@ final class TermVectorsTermsWriter exten
synchronized void flush(Map threadsAndFields, final SegmentWriteState state) throws IOException {
+ if (state.numDocsInStore > 0) {
+ // It's possible that all documents seen in this segment
+ // hit non-aborting exceptions, in which case we will
+ // not have yet init'd the TermVectorsWriter:
+ initTermVectorsWriter();
+ }
+
if (tvx != null) {
if (state.numDocsInStore > 0)