You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2018/09/05 10:52:32 UTC
lucene-solr:branch_7x: LUCENE-8484: Drop fully deleted reader in
SubReaderWrapper (#445)
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x e03dabb74 -> adfe98371
LUCENE-8484: Drop fully deleted reader in SubReaderWrapper (#445)
Today we can only wrap readers in SubReaderWrapper but never filter them out
entirely. This causes a invariant for soft-deletes that exposes fully deleted
segments with SoftDeletesDirectoryReaderWrapper. This change drops fully
deleted readers after they are wrapped.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/adfe9837
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/adfe9837
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/adfe9837
Branch: refs/heads/branch_7x
Commit: adfe98371d803ad97bf4cbb9ff31f65160a9c560
Parents: e03dabb
Author: Simon Willnauer <si...@apache.org>
Authored: Tue Sep 4 15:09:39 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Wed Sep 5 12:45:42 2018 +0200
----------------------------------------------------------------------
.../lucene/index/FilterDirectoryReader.java | 13 +++--
.../TestSoftDeletesDirectoryReaderWrapper.java | 51 ++++++++++++++++++++
2 files changed, 60 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/adfe9837/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
index 7003df5..1efd482 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -51,11 +52,15 @@ public abstract class FilterDirectoryReader extends DirectoryReader {
public static abstract class SubReaderWrapper {
private LeafReader[] wrap(List<? extends LeafReader> readers) {
- LeafReader[] wrapped = new LeafReader[readers.size()];
- for (int i = 0; i < readers.size(); i++) {
- wrapped[i] = wrap(readers.get(i));
+ List<LeafReader> wrapped = new ArrayList<>(readers.size());
+ for (LeafReader reader : readers) {
+ LeafReader wrap = wrap(reader);
+ assert wrap != null;
+ if (wrap.numDocs() > 0) {
+ wrapped.add(wrap);
+ }
}
- return wrapped;
+ return wrapped.toArray(new LeafReader[0]);
}
/** Constructor */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/adfe9837/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java
index d7a7999..a268650 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
@@ -34,6 +35,56 @@ import org.apache.lucene.util.LuceneTestCase;
public class TestSoftDeletesDirectoryReaderWrapper extends LuceneTestCase {
+ public void testDropFullyDeletedSegments() throws IOException {
+ IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+ String softDeletesField = "soft_delete";
+ indexWriterConfig.setSoftDeletesField(softDeletesField);
+ indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy(softDeletesField, MatchAllDocsQuery::new,
+ NoMergePolicy.INSTANCE));
+ try (Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, indexWriterConfig)) {
+
+ Document doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ doc.add(new StringField("version", "1", Field.Store.YES));
+ writer.addDocument(doc);
+ writer.commit();
+ doc = new Document();
+ doc.add(new StringField("id", "2", Field.Store.YES));
+ doc.add(new StringField("version", "1", Field.Store.YES));
+ writer.addDocument(doc);
+ writer.commit();
+
+ try (DirectoryReader reader = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(dir), softDeletesField)) {
+ assertEquals(2, reader.leaves().size());
+ assertEquals(2, reader.numDocs());
+ assertEquals(2, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ }
+ writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField(softDeletesField, 1));
+ writer.commit();
+ try (DirectoryReader reader = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(writer), softDeletesField)) {
+ assertEquals(1, reader.numDocs());
+ assertEquals(1, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ assertEquals(1, reader.leaves().size());
+ }
+ try (DirectoryReader reader = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(dir), softDeletesField)) {
+ assertEquals(1, reader.numDocs());
+ assertEquals(1, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ assertEquals(1, reader.leaves().size());
+ }
+
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ assertEquals(2, reader.numDocs());
+ assertEquals(2, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ assertEquals(2, reader.leaves().size());
+ }
+ }
+ }
+
public void testReuseUnchangedLeafReader() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig indexWriterConfig = newIndexWriterConfig();