You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/09/08 18:52:14 UTC
svn commit: r1166791 - in /lucene/dev/branches/branch_3x/lucene:
backwards/src/test/org/apache/lucene/index/
src/java/org/apache/lucene/index/ src/test/org/apache/lucene/index/
Author: mikemccand
Date: Thu Sep 8 16:52:14 2011
New Revision: 1166791
URL: http://svn.apache.org/viewvc?rev=1166791&view=rev
Log:
LUCENE-3189: make sure we sync on IW when invoking IFD
Modified:
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1166791&r1=1166790&r2=1166791&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Thu Sep 8 16:52:14 2011
@@ -795,7 +795,7 @@ class DirectoryReader extends IndexReade
// KeepOnlyLastCommitDeleter:
IndexFileDeleter deleter = new IndexFileDeleter(directory,
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
- segmentInfos, null);
+ segmentInfos, null, null);
segmentInfos.updateGeneration(deleter.getLastSegmentInfos());
segmentInfos.changed();
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1166791&r1=1166790&r2=1166791&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java Thu Sep 8 16:52:14 2011
@@ -101,6 +101,9 @@ final class IndexFileDeleter {
* infoStream != null */
public static boolean VERBOSE_REF_COUNTS = false;
+ // Used only for assert
+ private final IndexWriter writer;
+
void setInfoStream(PrintStream infoStream) {
this.infoStream = infoStream;
if (infoStream != null) {
@@ -112,6 +115,11 @@ final class IndexFileDeleter {
infoStream.println("IFD [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message);
}
+ // called only from assert
+ private boolean locked() {
+ return writer == null || Thread.holdsLock(writer);
+ }
+
/**
* Initialize the deleter: find all previous commits in
* the Directory, incref the files they reference, call
@@ -120,10 +128,11 @@ final class IndexFileDeleter {
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream)
+ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, IndexWriter writer)
throws CorruptIndexException, IOException {
this.infoStream = infoStream;
+ this.writer = writer;
final String currentSegmentsFile = segmentInfos.getCurrentSegmentFileName();
@@ -316,6 +325,8 @@ final class IndexFileDeleter {
* that segment.
*/
public void refresh(String segmentName) throws IOException {
+ assert locked();
+
String[] files = directory.listAll();
IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
String segmentPrefix1;
@@ -347,12 +358,14 @@ final class IndexFileDeleter {
// Set to null so that we regenerate the list of pending
// files; else we can accumulate same file more than
// once
+ assert locked();
deletable = null;
refresh(null);
}
public void close() throws IOException {
// DecRef old files from the last checkpoint, if any:
+ assert locked();
int size = lastFiles.size();
if (size > 0) {
for(int i=0;i<size;i++) {
@@ -374,6 +387,7 @@ final class IndexFileDeleter {
* unused commits again.
*/
void revisitPolicy() throws IOException {
+ assert locked();
if (infoStream != null) {
message("now revisitPolicy");
}
@@ -385,6 +399,7 @@ final class IndexFileDeleter {
}
public void deletePendingFiles() throws IOException {
+ assert locked();
if (deletable != null) {
List<String> oldDeletable = deletable;
deletable = null;
@@ -419,6 +434,7 @@ final class IndexFileDeleter {
* removed, we decref their files as well.
*/
public void checkpoint(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
+ assert locked();
if (infoStream != null) {
message("now checkpoint \"" + segmentInfos.getCurrentSegmentFileName() + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]");
@@ -453,6 +469,7 @@ final class IndexFileDeleter {
}
void incRef(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
+ assert locked();
// If this is a commit point, also incRef the
// segments_N file:
for( final String fileName: segmentInfos.files(directory, isCommit) ) {
@@ -461,12 +478,14 @@ final class IndexFileDeleter {
}
void incRef(Collection<String> files) throws IOException {
+ assert locked();
for(final String file : files) {
incRef(file);
}
}
void incRef(String fileName) throws IOException {
+ assert locked();
RefCount rc = getRefCount(fileName);
if (infoStream != null && VERBOSE_REF_COUNTS) {
message(" IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
@@ -475,12 +494,14 @@ final class IndexFileDeleter {
}
void decRef(Collection<String> files) throws IOException {
+ assert locked();
for(final String file : files) {
decRef(file);
}
}
void decRef(String fileName) throws IOException {
+ assert locked();
RefCount rc = getRefCount(fileName);
if (infoStream != null && VERBOSE_REF_COUNTS) {
message(" DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
@@ -494,12 +515,14 @@ final class IndexFileDeleter {
}
void decRef(SegmentInfos segmentInfos) throws IOException {
+ assert locked();
for (final String file : segmentInfos.files(directory, false)) {
decRef(file);
}
}
public boolean exists(String fileName) {
+ assert locked();
if (!refCounts.containsKey(fileName)) {
return false;
} else {
@@ -508,6 +531,7 @@ final class IndexFileDeleter {
}
private RefCount getRefCount(String fileName) {
+ assert locked();
RefCount rc;
if (!refCounts.containsKey(fileName)) {
rc = new RefCount(fileName);
@@ -519,6 +543,7 @@ final class IndexFileDeleter {
}
void deleteFiles(List<String> files) throws IOException {
+ assert locked();
for(final String file: files) {
deleteFile(file);
}
@@ -527,6 +552,7 @@ final class IndexFileDeleter {
/** Deletes the specified files, but only if they are new
* (have not yet been incref'd). */
void deleteNewFiles(Collection<String> files) throws IOException {
+ assert locked();
for (final String fileName: files) {
if (!refCounts.containsKey(fileName)) {
if (infoStream != null) {
@@ -539,6 +565,7 @@ final class IndexFileDeleter {
void deleteFile(String fileName)
throws IOException {
+ assert locked();
try {
if (infoStream != null) {
message("delete \"" + fileName + "\"");
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1166791&r1=1166790&r2=1166791&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Thu Sep 8 16:52:14 2011
@@ -1178,9 +1178,12 @@ public class IndexWriter implements Clos
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
- deleter = new IndexFileDeleter(directory,
- conf.getIndexDeletionPolicy(),
- segmentInfos, infoStream);
+ synchronized(this) {
+ deleter = new IndexFileDeleter(directory,
+ conf.getIndexDeletionPolicy(),
+ segmentInfos, infoStream,
+ this);
+ }
if (deleter.startingCommitDeleted) {
// Deletion policy deleted the "head" commit point.
@@ -3109,7 +3112,9 @@ public class IndexWriter implements Clos
// delete new non cfs files directly: they were never
// registered with IFD
- deleter.deleteNewFiles(info.files());
+ synchronized(this) {
+ deleter.deleteNewFiles(info.files());
+ }
info.setUseCompoundFile(true);
}
@@ -3380,34 +3385,45 @@ public class IndexWriter implements Clos
message("prepareCommit: flush");
ensureOpen(false);
- final boolean anySegmentsFlushed;
- final SegmentInfos toCommit;
- synchronized (this) {
- anySegmentsFlushed = doFlush(true);
- readerPool.commit(segmentInfos);
- // Must clone the segmentInfos while we still
- // hold fullFlushLock and while sync'd so that
- // no partial changes (eg a delete w/o
- // corresponding add from an updateDocument) can
- // sneak into the commit point:
- toCommit = (SegmentInfos) segmentInfos.clone();
- pendingCommitChangeCount = changeCount;
- // This protects the segmentInfos we are now going
- // to commit. This is important in case, eg, while
- // we are trying to sync all referenced files, a
- // merge completes which would otherwise have
- // removed the files we are now syncing.
- deleter.incRef(toCommit, false);
- }
+ boolean anySegmentsFlushed = false;
+ SegmentInfos toCommit = null;
boolean success = false;
try {
+ try {
+ synchronized (this) {
+ anySegmentsFlushed = doFlush(true);
+ readerPool.commit(segmentInfos);
+ toCommit = (SegmentInfos) segmentInfos.clone();
+ pendingCommitChangeCount = changeCount;
+ // This protects the segmentInfos we are now going
+ // to commit. This is important in case, eg, while
+ // we are trying to sync all referenced files, a
+ // merge completes which would otherwise have
+ // removed the files we are now syncing.
+ deleter.incRef(toCommit, false);
+ }
+ success = true;
+ } finally {
+ if (!success && infoStream != null) {
+ message("hit exception during prepareCommit");
+ }
+ doAfterFlush();
+ }
+ } catch (OutOfMemoryError oom) {
+ handleOOM(oom, "prepareCommit");
+ }
+
+ success = false;
+ try {
if (anySegmentsFlushed) {
maybeMerge();
}
success = true;
} finally {
if (!success) {
- deleter.decRef(toCommit);
+ synchronized (this) {
+ deleter.decRef(toCommit);
+ }
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1166791&r1=1166790&r2=1166791&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Thu Sep 8 16:52:14 2011
@@ -1320,6 +1320,7 @@ public class TestIndexWriter extends Luc
IndexWriterConfig conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2);
w = new IndexWriter(dir, conf);
+ w.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
doc.add(newField("field", "some text contents", Field.Store.YES, Field.Index.ANALYZED));
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1166791&r1=1166790&r2=1166791&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Thu Sep 8 16:52:14 2011
@@ -849,7 +849,7 @@ public class TestIndexWriterDelete exten
String[] startFiles = dir.listAll();
SegmentInfos infos = new SegmentInfos();
infos.read(dir);
- new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null);
+ new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
String[] endFiles = dir.listAll();
modifier.close();
dir.close();