You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2019/05/28 13:31:10 UTC
[lucene-solr] branch branch_7_7 updated (9e200b1 -> d4c30fc)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a change to branch branch_7_7
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 9e200b1 Revert "LUCENE-8809: Ensure release segment states"
new f27f31e Revert "Revert "LUCENE-8809: Ensure release segment states""
new d4c30fc LUCENE-8809: Changes entry for branch_7_7
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 2 +
.../apache/lucene/index/FrozenBufferedUpdates.java | 25 +++++------
.../org/apache/lucene/index/TestIndexWriter.java | 49 ++++++++++++++++++++++
3 files changed, 64 insertions(+), 12 deletions(-)
[lucene-solr] 02/02: LUCENE-8809: Changes entry for branch_7_7
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch branch_7_7
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit d4c30fc2856154f2c1fefc589eb7cd070a415b94
Author: Jan Høydahl <ja...@apache.org>
AuthorDate: Tue May 28 13:08:48 2019 +0200
LUCENE-8809: Changes entry for branch_7_7
---
lucene/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 065db66..d656cf5 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -26,6 +26,8 @@ Bug fixes
* LUCENE-8720: NameIntCacheLRU (in the facets module) had an int
overflow bug that disabled cleaning of the cache (Russell A Brown)
+* LUCENE-8809: Refresh and rollback concurrently can leave segment states unclosed (Nhat Nguyen)
+
======================= Lucene 7.7.1 =======================
(No changes)
[lucene-solr] 01/02: Revert "Revert "LUCENE-8809: Ensure release
segment states""
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch branch_7_7
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit f27f31e21816049bfcb4bda6d7d8231962c8c874
Author: Jan Høydahl <ja...@apache.org>
AuthorDate: Tue May 28 13:07:13 2019 +0200
Revert "Revert "LUCENE-8809: Ensure release segment states""
This reverts commit 9e200b16
I.e. we put the feature back into the branch by reverting the revert :)
---
.../apache/lucene/index/FrozenBufferedUpdates.java | 25 +++++------
.../org/apache/lucene/index/TestIndexWriter.java | 49 ++++++++++++++++++++++
2 files changed, 62 insertions(+), 12 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
index 121d0a4..c7c034c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -345,21 +344,23 @@ final class FrozenBufferedUpdates {
public static BufferedUpdatesStream.ApplyDeletesResult closeSegmentStates(IndexWriter writer, BufferedUpdatesStream.SegmentState[] segStates, boolean success) throws IOException {
List<SegmentCommitInfo> allDeleted = null;
long totDelCount = 0;
- final List<BufferedUpdatesStream.SegmentState> segmentStates = Arrays.asList(segStates);
- for (BufferedUpdatesStream.SegmentState segState : segmentStates) {
- if (success) {
- totDelCount += segState.rld.getDelCount() - segState.startDelCount;
- int fullDelCount = segState.rld.getDelCount();
- assert fullDelCount <= segState.rld.info.info.maxDoc() : fullDelCount + " > " + segState.rld.info.info.maxDoc();
- if (segState.rld.isFullyDeleted() && writer.getConfig().getMergePolicy().keepFullyDeletedSegment(() -> segState.reader) == false) {
- if (allDeleted == null) {
- allDeleted = new ArrayList<>();
+ try {
+ for (BufferedUpdatesStream.SegmentState segState : segStates) {
+ if (success) {
+ totDelCount += segState.rld.getDelCount() - segState.startDelCount;
+ int fullDelCount = segState.rld.getDelCount();
+ assert fullDelCount <= segState.rld.info.info.maxDoc() : fullDelCount + " > " + segState.rld.info.info.maxDoc();
+ if (segState.rld.isFullyDeleted() && writer.getConfig().getMergePolicy().keepFullyDeletedSegment(() -> segState.reader) == false) {
+ if (allDeleted == null) {
+ allDeleted = new ArrayList<>();
+ }
+ allDeleted.add(segState.reader.getOriginalSegmentInfo());
}
- allDeleted.add(segState.reader.getOriginalSegmentInfo());
}
}
+ } finally {
+ IOUtils.close(segStates);
}
- IOUtils.close(segmentStates);
if (writer.infoStream.isEnabled("BD")) {
writer.infoStream.message("BD", "closeSegmentStates: " + totDelCount + " new deleted documents; pool " + writer.getPendingUpdatesCount()+ " packets; bytesUsed=" + writer.getReaderPoolRamBytesUsed());
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 7e8b7ac..e8d8ebe 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -39,6 +39,7 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -76,6 +77,8 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.AlreadyClosedException;
@@ -3753,4 +3756,50 @@ public class TestIndexWriter extends LuceneTestCase {
}
}
+ public void testRefreshAndRollbackConcurrently() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+ AtomicBoolean stopped = new AtomicBoolean();
+ Semaphore indexedDocs = new Semaphore(0);
+ Thread indexer = new Thread(() -> {
+ while (stopped.get() == false) {
+ try {
+ String id = Integer.toString(random().nextInt(100));
+ Document doc = new Document();
+ doc.add(new StringField("id", id, Field.Store.YES));
+ w.updateDocument(new Term("id", id), doc);
+ indexedDocs.release(1);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ } catch (AlreadyClosedException ignored) {
+ return;
+ }
+ }
+ });
+
+ SearcherManager sm = new SearcherManager(w, new SearcherFactory());
+ Thread refresher = new Thread(() -> {
+ while (stopped.get() == false) {
+ try {
+ sm.maybeRefreshBlocking();
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ } catch (AlreadyClosedException ignored) {
+ return;
+ }
+ }
+ });
+
+ try {
+ indexer.start();
+ refresher.start();
+ indexedDocs.acquire(1 + random().nextInt(100));
+ w.rollback();
+ } finally {
+ stopped.set(true);
+ indexer.join();
+ refresher.join();
+ IOUtils.close(sm, dir);
+ }
+ }
}