You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2018/05/21 18:32:10 UTC
lucene-solr:branch_7x: LUCENE-8324: Checkpoint after fully deletes
segment is dropped on flush
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x f0d6a0e63 -> cc2ee2305
LUCENE-8324: Checkpoint after fully deletes segment is dropped on flush
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cc2ee230
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cc2ee230
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cc2ee230
Branch: refs/heads/branch_7x
Commit: cc2ee2305001a49536886653d2133ee1a3b51b82
Parents: f0d6a0e
Author: Simon Willnauer <si...@apache.org>
Authored: Sun May 20 14:06:14 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon May 21 20:31:54 2018 +0200
----------------------------------------------------------------------
.../org/apache/lucene/index/IndexWriter.java | 1 +
.../apache/lucene/index/TestIndexWriter.java | 36 ++++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc2ee230/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index a28b9bf..e1ea56f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -2652,6 +2652,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
try {
if (isFullyDeleted(rld)) {
dropDeletedSegment(newSegment);
+ checkpoint();
}
} finally {
release(rld);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc2ee230/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index c93007a..5f6171f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -40,6 +40,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.analysis.Analyzer;
@@ -3337,4 +3338,39 @@ public class TestIndexWriter extends LuceneTestCase {
t.join();
}
}
+
+ private static void assertFiles(IndexWriter writer) throws IOException {
+ // remove segment files we don't know if we have committed and what is kept around
+ Set<String> segFiles = new HashSet<>(writer.segmentInfos.files(true)).stream()
+ .filter(f -> f.startsWith("segments") == false).collect(Collectors.toSet());
+ Set<String> dirFiles = new HashSet<>(Arrays.asList(writer.getDirectory().listAll()))
+ .stream().filter(f -> f.startsWith("segments") == false).collect(Collectors.toSet());
+ Set<String> s = new HashSet<>(segFiles);
+ s.removeAll(dirFiles);
+ assertEquals(segFiles.toString() + " vs "+ dirFiles.toString(), segFiles.size(), dirFiles.size());
+ }
+
+ public void testFullyDeletedSegmentsReleaseFiles() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = newIndexWriterConfig();
+ config.setRAMBufferSizeMB(Integer.MAX_VALUE);
+ config.setMaxBufferedDocs(2); // no auto flush
+ IndexWriter writer = new IndexWriter(dir, config);
+ Document d = new Document();
+ d.add(new StringField("id", "doc-0", Field.Store.YES));
+ writer.addDocument(d);
+ writer.flush();
+ d = new Document();
+ d.add(new StringField("id", "doc-1", Field.Store.YES));
+ writer.addDocument(d);
+ writer.deleteDocuments(new Term("id", "doc-1"));
+ assertEquals(1, writer.segmentInfos.asList().size());
+ writer.flush();
+ assertEquals(1, writer.segmentInfos.asList().size());
+ writer.commit();
+ assertFiles(writer);
+ assertEquals(1, writer.segmentInfos.asList().size());
+ IOUtils.close(writer, dir);
+ }
+
}