You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/05/07 16:33:47 UTC
svn commit: r1593023 -
/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
Author: mikemccand
Date: Wed May 7 14:33:47 2014
New Revision: 1593023
URL: http://svn.apache.org/r1593023
Log:
LUCENE-5644: add test case
Added:
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java (with props)
Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java?rev=1593023&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java Wed May 7 14:33:47 2014
@@ -0,0 +1,230 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+public class TestIndexWriterThreadsToSegments extends LuceneTestCase {
+
+ // LUCENE-5644: for first segment, two threads each indexed one doc (likely concurrently), but for second segment, each thread indexed the
+ // doc NOT at the same time, and should have shared the same thread state / segment
+ public void testSegmentCountOnFlushBasic() throws Exception {
+ Directory dir = newDirectory();
+ final IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ final CountDownLatch startingGun = new CountDownLatch(1);
+ final CountDownLatch startDone = new CountDownLatch(2);
+ final CountDownLatch middleGun = new CountDownLatch(1);
+ final CountDownLatch finalGun = new CountDownLatch(1);
+ Thread[] threads = new Thread[2];
+ for(int i=0;i<threads.length;i++) {
+ final int threadID = i;
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ try {
+ startingGun.await();
+ Document doc = new Document();
+ doc.add(newTextField("field", "here is some text", Field.Store.NO));
+ w.addDocument(doc);
+ startDone.countDown();
+
+ middleGun.await();
+ if (threadID == 0) {
+ w.addDocument(doc);
+ } else {
+ finalGun.await();
+ w.addDocument(doc);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ threads[i].start();
+ }
+
+ startingGun.countDown();
+ startDone.await();
+
+ IndexReader r = DirectoryReader.open(w, true);
+ assertEquals(2, r.numDocs());
+ int numSegments = r.leaves().size();
+ // 1 segment if the threads ran sequentially, else 2:
+ assertTrue(numSegments <= 2);
+ r.close();
+
+ middleGun.countDown();
+ threads[0].join();
+
+ finalGun.countDown();
+ threads[1].join();
+
+ r = DirectoryReader.open(w, true);
+ assertEquals(4, r.numDocs());
+ // Both threads should have shared a single thread state since they did not try to index concurrently:
+ assertEquals(1+numSegments, r.leaves().size());
+ r.close();
+
+ w.close();
+ dir.close();
+ }
+
+ /** Maximum number of simultaneous threads to use for each iteration. */
+ private static final int MAX_THREADS_AT_ONCE = 10;
+
+ static class CheckSegmentCount implements Runnable, Closeable {
+ private final IndexWriter w;
+ private final AtomicInteger maxThreadCountPerIter;
+ private final AtomicInteger indexingCount;
+ private DirectoryReader r;
+
+ public CheckSegmentCount(IndexWriter w, AtomicInteger maxThreadCountPerIter, AtomicInteger indexingCount) throws IOException {
+ this.w = w;
+ this.maxThreadCountPerIter = maxThreadCountPerIter;
+ this.indexingCount = indexingCount;
+ r = DirectoryReader.open(w, true);
+ assertEquals(0, r.leaves().size());
+ setNextIterThreadCount();
+ }
+
+ @Override
+ public void run() {
+ try {
+ int oldSegmentCount = r.leaves().size();
+ DirectoryReader r2 = DirectoryReader.openIfChanged(r);
+ assertNotNull(r2);
+ r.close();
+ r = r2;
+ int maxThreadStates = w.getConfig().getMaxThreadStates();
+ int maxExpectedSegments = oldSegmentCount + Math.min(maxThreadStates, maxThreadCountPerIter.get());
+ if (VERBOSE) {
+ System.out.println("TEST: iter done; now verify oldSegCount=" + oldSegmentCount + " newSegCount=" + r2.leaves().size() + " maxExpected=" + maxExpectedSegments);
+ }
+ // NOTE: it won't necessarily be ==, in case some threads were strangely scheduled and never conflicted with one another (should be uncommon...?):
+ assertTrue(r.leaves().size() <= maxExpectedSegments);
+ setNextIterThreadCount();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void setNextIterThreadCount() {
+ indexingCount.set(0);
+ maxThreadCountPerIter.set(TestUtil.nextInt(random(), 1, MAX_THREADS_AT_ONCE));
+ if (VERBOSE) {
+ System.out.println("TEST: iter set maxThreadCount=" + maxThreadCountPerIter.get());
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ r.close();
+ r = null;
+ }
+ }
+
+ // LUCENE-5644: index docs w/ multiple threads but in between flushes we limit how many threads can index concurrently in the next
+ // iteration, and then verify that no more segments were flushed than number of threads:
+ public void testSegmentCountOnFlushRandom() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+
+ int maxThreadStates = TestUtil.nextInt(random(), 1, 12);
+
+ if (VERBOSE) {
+ System.out.println("TEST: maxThreadStates=" + maxThreadStates);
+ }
+
+ // Never trigger flushes (so we only flush on getReader):
+ iwc.setMaxBufferedDocs(100000000);
+ iwc.setRAMBufferSizeMB(-1);
+ iwc.setMaxThreadStates(maxThreadStates);
+
+ // Never trigger merges (so we can simplistically count flushed segments):
+ iwc.setMergePolicy(NoMergePolicy.INSTANCE);
+
+ final IndexWriter w = new IndexWriter(dir, iwc);
+
+ // How many threads are indexing in the current cycle:
+ final AtomicInteger indexingCount = new AtomicInteger();
+
+ // How many threads we will use on each cycle:
+ final AtomicInteger maxThreadCount = new AtomicInteger();
+
+ CheckSegmentCount checker = new CheckSegmentCount(w, maxThreadCount, indexingCount);
+
+ // We spin up 10 threads up front, but then in between flushes we limit how many can run on each iteration
+ final int ITERS = 100;
+ Thread[] threads = new Thread[MAX_THREADS_AT_ONCE];
+
+ // We use this to stop all threads once they've indexed their docs in the current iter, and pull a new NRT reader, and verify the
+ // segment count:
+ final CyclicBarrier barrier = new CyclicBarrier(MAX_THREADS_AT_ONCE, checker);
+
+ for(int i=0;i<threads.length;i++) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ try {
+ for(int iter=0;iter<ITERS;iter++) {
+ if (indexingCount.incrementAndGet() <= maxThreadCount.get()) {
+ if (VERBOSE) {
+ System.out.println("TEST: " + Thread.currentThread().getName() + ": do index");
+ }
+
+ // We get to index on this cycle:
+ Document doc = new Document();
+ doc.add(newTextField("field", "here is some text that is a bit longer than normal trivial text", Field.Store.NO));
+ for(int j=0;j<200;j++) {
+ w.addDocument(doc);
+ }
+ } else {
+ // We lose: no indexing for us on this cycle
+ if (VERBOSE) {
+ System.out.println("TEST: " + Thread.currentThread().getName() + ": don't index");
+ }
+ }
+ barrier.await();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ threads[i].start();
+ }
+
+ for(int i=0;i<threads.length;i++) {
+ threads[i].join();
+ }
+
+ IOUtils.close(checker, w, dir);
+ }
+}