You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2018/05/21 18:38:29 UTC
lucene-solr:master: LUCENE-8324: Checkpoint after fully deletes
segment is dropped on flush
Repository: lucene-solr
Updated Branches:
refs/heads/master 669b9e7a5 -> 2ce53791d
LUCENE-8324: Checkpoint after fully deletes segment is dropped on flush
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2ce53791
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2ce53791
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2ce53791
Branch: refs/heads/master
Commit: 2ce53791d3205efff5eb12d0d24911b3ea31abe3
Parents: 669b9e7
Author: Simon Willnauer <si...@apache.org>
Authored: Sun May 20 14:06:14 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon May 21 20:32:37 2018 +0200
----------------------------------------------------------------------
.../org/apache/lucene/index/IndexWriter.java | 1 +
.../apache/lucene/index/TestIndexWriter.java | 36 ++++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2ce53791/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index af2fd47..8db29ff 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -2650,6 +2650,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
try {
if (isFullyDeleted(rld)) {
dropDeletedSegment(newSegment);
+ checkpoint();
}
} finally {
release(rld);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2ce53791/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 1dd1eb2..a36920b 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -40,6 +40,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.analysis.Analyzer;
@@ -3314,4 +3315,39 @@ public class TestIndexWriter extends LuceneTestCase {
t.join();
}
}
+
+ private static void assertFiles(IndexWriter writer) throws IOException {
+ // remove segment files we don't know if we have committed and what is kept around
+ Set<String> segFiles = new HashSet<>(writer.segmentInfos.files(true)).stream()
+ .filter(f -> f.startsWith("segments") == false).collect(Collectors.toSet());
+ Set<String> dirFiles = new HashSet<>(Arrays.asList(writer.getDirectory().listAll()))
+ .stream().filter(f -> f.startsWith("segments") == false).collect(Collectors.toSet());
+ Set<String> s = new HashSet<>(segFiles);
+ s.removeAll(dirFiles);
+ assertEquals(segFiles.toString() + " vs "+ dirFiles.toString(), segFiles.size(), dirFiles.size());
+ }
+
+ public void testFullyDeletedSegmentsReleaseFiles() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = newIndexWriterConfig();
+ config.setRAMBufferSizeMB(Integer.MAX_VALUE);
+ config.setMaxBufferedDocs(2); // no auto flush
+ IndexWriter writer = new IndexWriter(dir, config);
+ Document d = new Document();
+ d.add(new StringField("id", "doc-0", Field.Store.YES));
+ writer.addDocument(d);
+ writer.flush();
+ d = new Document();
+ d.add(new StringField("id", "doc-1", Field.Store.YES));
+ writer.addDocument(d);
+ writer.deleteDocuments(new Term("id", "doc-1"));
+ assertEquals(1, writer.segmentInfos.asList().size());
+ writer.flush();
+ assertEquals(1, writer.segmentInfos.asList().size());
+ writer.commit();
+ assertFiles(writer);
+ assertEquals(1, writer.segmentInfos.asList().size());
+ IOUtils.close(writer, dir);
+ }
+
}