You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/12/17 14:51:53 UTC
svn commit: r1646240 - in /lucene/dev/trunk/lucene:
core/src/java/org/apache/lucene/index/ core/src/test/org/apache/lucene/index/
test-framework/src/java/org/apache/lucene/index/
Author: rmuir
Date: Wed Dec 17 13:51:53 2014
New Revision: 1646240
URL: http://svn.apache.org/r1646240
Log:
LUCENE-6117: make infostream usable again
Added:
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java (with props)
Modified:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1646240&r1=1646239&r2=1646240&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Wed Dec 17 13:51:53 2014
@@ -388,7 +388,7 @@ final class DocumentsWriter implements C
writer.globalFieldNumberMap);
state.dwpt = new DocumentsWriterPerThread(writer.newSegmentName(),
directory, config, infoStream, deleteQueue, infos,
- writer.pendingNumDocs);
+ writer.pendingNumDocs, writer.enableTestPoints);
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1646240&r1=1646239&r2=1646240&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Wed Dec 17 13:51:53 2014
@@ -155,9 +155,10 @@ class DocumentsWriterPerThread {
final IntBlockPool.Allocator intBlockAllocator;
private final AtomicLong pendingNumDocs;
private final LiveIndexWriterConfig indexWriterConfig;
+ private final boolean enableTestPoints;
public DocumentsWriterPerThread(String segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue,
- FieldInfos.Builder fieldInfos, AtomicLong pendingNumDocs) throws IOException {
+ FieldInfos.Builder fieldInfos, AtomicLong pendingNumDocs, boolean enableTestPoints) throws IOException {
this.directoryOrig = directory;
this.directory = new TrackingDirectoryWrapper(directory);
this.fieldInfos = fieldInfos;
@@ -184,6 +185,7 @@ class DocumentsWriterPerThread {
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
consumer = indexWriterConfig.getIndexingChain().getChain(this);
+ this.enableTestPoints = enableTestPoints;
}
public FieldInfos.Builder getFieldInfosBuilder() {
@@ -191,7 +193,8 @@ class DocumentsWriterPerThread {
}
final void testPoint(String message) {
- if (infoStream.isEnabled("TP")) {
+ if (enableTestPoints) {
+ assert infoStream.isEnabled("TP"); // don't enable unless you need them.
infoStream.message("TP", message);
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1646240&r1=1646239&r2=1646240&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Wed Dec 17 13:51:53 2014
@@ -214,6 +214,9 @@ public class IndexWriter implements Clos
static int getActualMaxDocs() {
return IndexWriter.actualMaxDocs;
}
+
+ /** Used only for testing. */
+ boolean enableTestPoints = false;
private static final int UNBOUNDED_MAX_MERGE_SEGMENTS = -1;
@@ -4417,7 +4420,8 @@ public class IndexWriter implements Clos
// startMergeInit
// DocumentsWriter.ThreadState.init start
private final void testPoint(String message) {
- if (infoStream.isEnabled("TP")) {
+ if (enableTestPoints) {
+ assert infoStream.isEnabled("TP"); // don't enable unless you need them.
infoStream.message("TP", message);
}
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1646240&r1=1646239&r2=1646240&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Dec 17 13:51:53 2014
@@ -2428,7 +2428,7 @@ public class TestIndexWriter extends Luc
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
final SetOnce<IndexWriter> iwRef = new SetOnce<>();
- iwc.setInfoStream(new RandomIndexWriter.TestPointInfoStream(iwc.getInfoStream(), new RandomIndexWriter.TestPoint() {
+ IndexWriter evilWriter = RandomIndexWriter.mockIndexWriter(dir, iwc, new RandomIndexWriter.TestPoint() {
@Override
public void apply(String message) {
if ("startCommitMerge".equals(message)) {
@@ -2437,8 +2437,7 @@ public class TestIndexWriter extends Luc
iwRef.get().setKeepFullyDeletedSegments(true);
}
}
- }));
- IndexWriter evilWriter = new IndexWriter(dir, iwc);
+ });
iwRef.set(evilWriter);
for (int i = 0; i < 1000; i++) {
addDoc(evilWriter);
@@ -2646,10 +2645,12 @@ public class TestIndexWriter extends Luc
final CountDownLatch startCommit = new CountDownLatch(1);
final CountDownLatch finishCommit = new CountDownLatch(1);
- // infostream that "takes a long time" to commit
- InfoStream slowCommittingInfoStream = new InfoStream() {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(null);
+ // use an infostream that "takes a long time" to commit
+ final IndexWriter iw = RandomIndexWriter.mockIndexWriter(dir, iwc, new RandomIndexWriter.TestPoint() {
@Override
- public void message(String component, String message) {
+ public void apply(String message) {
if (message.equals("finishStartCommit")) {
startCommit.countDown();
try {
@@ -2659,20 +2660,7 @@ public class TestIndexWriter extends Luc
}
}
}
-
- @Override
- public boolean isEnabled(String component) {
- return true;
- }
-
- @Override
- public void close() throws IOException {}
- };
-
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(null);
- iwc.setInfoStream(slowCommittingInfoStream);
- final IndexWriter iw = new IndexWriter(dir, iwc);
+ });
new Thread() {
@Override
public void run() {
Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java?rev=1646240&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java Wed Dec 17 13:51:53 2014
@@ -0,0 +1,84 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.LuceneTestCase;
+
+/** Tests indexwriter's infostream */
+public class TestInfoStream extends LuceneTestCase {
+
+ /** we shouldn't have test points unless we ask */
+ public void testTestPointsOff() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(null);
+ iwc.setInfoStream(new InfoStream() {
+ @Override
+ public void close() throws IOException {}
+
+ @Override
+ public void message(String component, String message) {
+ assertFalse("TP".equals(component));
+ }
+
+ @Override
+ public boolean isEnabled(String component) {
+ assertFalse("TP".equals(component));
+ return true;
+ }
+ });
+ IndexWriter iw = new IndexWriter(dir, iwc);
+ iw.addDocument(new Document());
+ iw.close();
+ dir.close();
+ }
+
+ /** but they should work when we need */
+ public void testTestPointsOn() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(null);
+ AtomicBoolean seenTestPoint = new AtomicBoolean();
+ iwc.setInfoStream(new InfoStream() {
+ @Override
+ public void close() throws IOException {}
+
+ @Override
+ public void message(String component, String message) {
+ if ("TP".equals(component)) {
+ seenTestPoint.set(true);
+ }
+ }
+
+ @Override
+ public boolean isEnabled(String component) {
+ return true;
+ }
+ });
+ IndexWriter iw = new IndexWriter(dir, iwc);
+ iw.enableTestPoints = true;
+ iw.addDocument(new Document());
+ iw.close();
+ dir.close();
+ assertTrue(seenTestPoint.get());
+ }
+}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1646240&r1=1646239&r2=1646240&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java Wed Dec 17 13:51:53 2014
@@ -45,16 +45,6 @@ public class TestStressIndexing2 extends
static int maxBufferedDocs=3;
static int seed=0;
- public final class YieldTestPoint implements RandomIndexWriter.TestPoint {
-
- @Override
- public void apply(String name) {
- // if (name.equals("startCommit")) {
- if (random().nextInt(4) == 2)
- Thread.yield();
- }
- }
-//
public void testRandomIWReader() throws Throwable {
Directory dir = newDirectory();
@@ -148,7 +138,7 @@ public class TestStressIndexing2 extends
.setOpenMode(OpenMode.CREATE)
.setRAMBufferSizeMB(0.1)
.setMaxBufferedDocs(maxBufferedDocs)
- .setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
+ .setMergePolicy(newLogMergePolicy()), random());
w.commit();
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setNoCFSRatio(0.0);
@@ -203,7 +193,7 @@ public class TestStressIndexing2 extends
.setMaxBufferedDocs(maxBufferedDocs)
.setIndexerThreadPool(new DocumentsWriterPerThreadPool(maxThreadStates))
.setReaderPooling(doReaderPooling)
- .setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
+ .setMergePolicy(newLogMergePolicy()), random());
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setNoCFSRatio(0.0);
lmp.setMergeFactor(mergeFactor);
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1646240&r1=1646239&r2=1646240&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Wed Dec 17 13:51:53 2014
@@ -50,7 +50,7 @@ public class RandomIndexWriter implement
private boolean getReaderCalled;
private final Codec codec; // sugar
-
+ /** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */
public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
// Randomly calls Thread.yield so we mixup thread scheduling
final Random random = new Random(r.nextLong());
@@ -63,9 +63,12 @@ public class RandomIndexWriter implement
});
}
+ /** Returns an indexwriter that enables the specified test point */
public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, TestPoint testPoint) throws IOException {
conf.setInfoStream(new TestPointInfoStream(conf.getInfoStream(), testPoint));
- return new IndexWriter(dir, conf);
+ IndexWriter iw = new IndexWriter(dir, conf);
+ iw.enableTestPoints = true;
+ return iw;
}
/** create a RandomIndexWriter with a random config: Uses MockAnalyzer */