You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/08/31 14:09:15 UTC
svn commit: r1163594 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/backwards/src/test-framework/
lucene/backwards/src/test/ lucene/src/java/org/apache/lucene/index/
lucene/src/test/org/apache/lucene/index/ solr/
Author: mikemccand
Date: Wed Aug 31 12:09:14 2011
New Revision: 1163594
URL: http://svn.apache.org/viewvc?rev=1163594&view=rev
Log:
LUCENE-3409: drop reader pool from IW.deleteAll
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1163594&r1=1163593&r2=1163594&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Wed Aug 31 12:09:14 2011
@@ -54,6 +54,10 @@ Bug fixes
* LUCENE-3402: term vectors disappeared from the index if optimize() was called
following addIndexes(). (Shai Erera)
+* LUCENE-3409: IndexWriter.deleteAll was failing to close pooled NRT
+ SegmentReaders, leading to unused files accumulating in the
+ Directory. (tal steier via Mike McCandless)
+
New Features
* LUCENE-3290: Added FieldInvertState.numUniqueTerms
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1163594&r1=1163593&r2=1163594&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java Wed Aug 31 12:09:14 2011
@@ -344,6 +344,10 @@ final class IndexFileDeleter {
}
public void refresh() throws IOException {
+ // Set to null so that we regenerate the list of pending
+ // files; else we can accumulate same file more than
+ // once
+ deletable = null;
refresh(null);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1163594&r1=1163593&r2=1163594&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed Aug 31 12:09:14 2011
@@ -27,7 +27,6 @@ import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -594,6 +593,18 @@ public class IndexWriter implements Clos
}
}
+ public synchronized void dropAll() throws IOException {
+ for(SegmentReader reader : readerMap.values()) {
+ reader.hasChanges = false;
+
+ // NOTE: it is allowed that this decRef does not
+ // actually close the SR; this can happen when a
+ // near real-time reader using this SR is still open
+ reader.decRef();
+ }
+ readerMap.clear();
+ }
+
/** Remove all our references to readers, and commits
* any pending changes. */
synchronized void close() throws IOException {
@@ -601,11 +612,8 @@ public class IndexWriter implements Clos
// sync'd on IW:
assert Thread.holdsLock(IndexWriter.this);
- Iterator<Map.Entry<SegmentInfo,SegmentReader>> iter = readerMap.entrySet().iterator();
- while (iter.hasNext()) {
+ for(Map.Entry<SegmentInfo,SegmentReader> ent : readerMap.entrySet()) {
- Map.Entry<SegmentInfo,SegmentReader> ent = iter.next();
-
SegmentReader sr = ent.getValue();
if (sr.hasChanges) {
assert infoIsLive(sr.getSegmentInfo());
@@ -617,14 +625,14 @@ public class IndexWriter implements Clos
deleter.checkpoint(segmentInfos, false);
}
- iter.remove();
-
// NOTE: it is allowed that this decRef does not
// actually close the SR; this can happen when a
// near real-time reader is kept open after the
// IndexWriter instance is closed
sr.decRef();
}
+
+ readerMap.clear();
}
/**
@@ -2909,7 +2917,7 @@ public class IndexWriter implements Clos
deleter.refresh();
// Don't bother saving any changes in our segmentInfos
- readerPool.clear(null);
+ readerPool.dropAll();
// Mark that the index has changed
++changeCount;
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1163594&r1=1163593&r2=1163594&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Aug 31 12:09:14 2011
@@ -30,12 +30,10 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Collections;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockFixedLengthPayloadFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
@@ -66,7 +64,6 @@ import org.apache.lucene.store.NoLockFac
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
-import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.ThreadInterruptedException;
@@ -1916,4 +1913,28 @@ public class TestIndexWriter extends Luc
r.close();
dir.close();
}
+
+ public void testDeleteAllNRTLeftoverFiles() throws Exception {
+
+ Directory d = new MockDirectoryWrapper(random, new RAMDirectory());
+ IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ Document doc = new Document();
+ for(int i = 0; i < 20; i++) {
+ for(int j = 0; j < 100; ++j) {
+ w.addDocument(doc);
+ }
+ w.commit();
+ IndexReader.open(w, true).close();
+
+ w.deleteAll();
+ w.commit();
+
+ // Make sure we accumulate no files except for empty
+ // segments_N and segments.gen:
+ assertTrue(d.listAll().length <= 2);
+ }
+
+ w.close();
+ d.close();
+ }
}