You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2018/05/23 05:56:03 UTC
[1/2] lucene-solr:branch_7x: LUCENE-8328: Ensure ReadersAndUpdates
consistently executes under lock
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x e0ccf8810 -> 3ed9f98ed
LUCENE-8328: Ensure ReadersAndUpdates consistently executes under lock
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/245d6cd6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/245d6cd6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/245d6cd6
Branch: refs/heads/branch_7x
Commit: 245d6cd6ad2c3babed1c48c8a79b598506f20f4f
Parents: e0ccf88
Author: Simon Willnauer <si...@apache.org>
Authored: Wed May 23 07:40:47 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Wed May 23 07:55:43 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../apache/lucene/index/ReadersAndUpdates.java | 3 +-
.../org/apache/lucene/index/TestReaderPool.java | 61 ++++++++++++++++++++
3 files changed, 66 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/245d6cd6/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f9fe92a..73efcce 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -131,6 +131,9 @@ Bug Fixes
* LUCENE-8320: Fix WindowsFS to correctly account for rename and hardlinks.
(Simon Willnauer, Nhat Nguyen)
+* LUCENE-8328: Ensure ReadersAndUpdates consistently executes under lock.
+ (Nhat Nguyen via Simon Willnauer)
+
Other
* LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/245d6cd6/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index 96740a7..848cea7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -249,7 +249,7 @@ final class ReadersAndUpdates {
return pendingDeletes.numDeletesToMerge(policy, this::getLatestReader);
}
- private CodecReader getLatestReader() throws IOException {
+ private synchronized CodecReader getLatestReader() throws IOException {
if (this.reader == null) {
// get a reader and dec the ref right away we just make sure we have a reader
getReader(IOContext.READ).decRef();
@@ -667,6 +667,7 @@ final class ReadersAndUpdates {
private SegmentReader createNewReaderWithLatestLiveDocs(SegmentReader reader) throws IOException {
assert reader != null;
+ assert Thread.holdsLock(this) : Thread.currentThread().getName();
SegmentReader newReader = new SegmentReader(info, reader, pendingDeletes.getLiveDocs(),
info.info.maxDoc() - info.getDelCount() - pendingDeletes.numPendingDeletes());
boolean success2 = false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/245d6cd6/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java b/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java
index 29c5dd3..19a6af2 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java
@@ -19,6 +19,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.document.Document;
@@ -28,6 +30,7 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.IOSupplier;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NullInfoStream;
@@ -205,6 +208,64 @@ public class TestReaderPool extends LuceneTestCase {
IOUtils.close(pool, reader, directory);
}
+ public void testPassReaderToMergePolicyConcurrently() throws Exception {
+ Directory directory = newDirectory();
+ FieldInfos.FieldNumbers fieldNumbers = buildIndex(directory);
+ StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(directory);
+ SegmentInfos segmentInfos = reader.segmentInfos.clone();
+ ReaderPool pool = new ReaderPool(directory, directory, segmentInfos, fieldNumbers, () -> 0L,
+ new NullInfoStream(), null, null);
+ if (random().nextBoolean()) {
+ pool.enableReaderPooling();
+ }
+ AtomicBoolean isDone = new AtomicBoolean();
+ CountDownLatch latch = new CountDownLatch(1);
+ Thread refresher = new Thread(() -> {
+ try {
+ latch.countDown();
+ while (isDone.get() == false) {
+ for (SegmentCommitInfo commitInfo : segmentInfos) {
+ ReadersAndUpdates readersAndUpdates = pool.get(commitInfo, true);
+ SegmentReader segmentReader = readersAndUpdates.getReader(IOContext.READ);
+ readersAndUpdates.release(segmentReader);
+ pool.release(readersAndUpdates, random().nextBoolean());
+ }
+ }
+ } catch (Exception ex) {
+ throw new AssertionError(ex);
+ }
+ });
+ refresher.start();
+ MergePolicy mergePolicy = new FilterMergePolicy(newMergePolicy()) {
+ @Override
+ public boolean keepFullyDeletedSegment(IOSupplier<CodecReader> readerIOSupplier) throws IOException {
+ CodecReader reader = readerIOSupplier.get();
+ assert reader.maxDoc() > 0; // just try to access the reader
+ return true;
+ }
+ };
+ latch.await();
+ for (int i = 0; i < reader.maxDoc(); i++) {
+ for (SegmentCommitInfo commitInfo : segmentInfos) {
+ ReadersAndUpdates readersAndUpdates = pool.get(commitInfo, true);
+ SegmentReader sr = readersAndUpdates.getReadOnlyClone(IOContext.READ);
+ PostingsEnum postings = sr.postings(new Term("id", "" + i));
+ sr.decRef();
+ if (postings != null) {
+ for (int docId = postings.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = postings.nextDoc()) {
+ readersAndUpdates.delete(docId);
+ assertTrue(readersAndUpdates.keepFullyDeletedSegment(mergePolicy));
+ }
+ }
+ assertTrue(readersAndUpdates.keepFullyDeletedSegment(mergePolicy));
+ pool.release(readersAndUpdates, random().nextBoolean());
+ }
+ }
+ isDone.set(true);
+ refresher.join();
+ IOUtils.close(pool, reader, directory);
+ }
+
private FieldInfos.FieldNumbers buildIndex(Directory directory) throws IOException {
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig());
for (int i = 0; i < 10; i++) {
[2/2] lucene-solr:branch_7x: LUCENE-8324: Fix test to exclude the
write.lock in expected files
Posted by si...@apache.org.
LUCENE-8324: Fix test to exclude the write.lock in expected files
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3ed9f98e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3ed9f98e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3ed9f98e
Branch: refs/heads/branch_7x
Commit: 3ed9f98ed8083716e24bf0aa5d72138da2d8b518
Parents: 245d6cd
Author: Simon Willnauer <si...@apache.org>
Authored: Wed May 23 07:43:53 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Wed May 23 07:55:52 2018 +0200
----------------------------------------------------------------------
.../core/src/test/org/apache/lucene/index/TestIndexWriter.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed9f98e/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 5f6171f..7ebfdd4 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -40,6 +40,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -3340,11 +3341,12 @@ public class TestIndexWriter extends LuceneTestCase {
}
private static void assertFiles(IndexWriter writer) throws IOException {
+ Predicate<String> filter = file -> file.startsWith("segments") == false && file.equals("write.lock") == false;
// remove segment files we don't know if we have committed and what is kept around
Set<String> segFiles = new HashSet<>(writer.segmentInfos.files(true)).stream()
- .filter(f -> f.startsWith("segments") == false).collect(Collectors.toSet());
+ .filter(filter).collect(Collectors.toSet());
Set<String> dirFiles = new HashSet<>(Arrays.asList(writer.getDirectory().listAll()))
- .stream().filter(f -> f.startsWith("segments") == false).collect(Collectors.toSet());
+ .stream().filter(filter).collect(Collectors.toSet());
Set<String> s = new HashSet<>(segFiles);
s.removeAll(dirFiles);
assertEquals(segFiles.toString() + " vs "+ dirFiles.toString(), segFiles.size(), dirFiles.size());