You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/12/17 14:55:47 UTC
svn commit: r1646242 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/index/ lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/index/
Author: rmuir
Date: Wed Dec 17 13:55:47 2014
New Revision: 1646242
URL: http://svn.apache.org/r1646242
Log:
LUCENE-6117: make infostream usable again
Added:
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java
- copied, changed from r1646240, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1646242&r1=1646241&r2=1646242&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Wed Dec 17 13:55:47 2014
@@ -389,7 +389,7 @@ final class DocumentsWriter implements C
writer.globalFieldNumberMap);
state.dwpt = new DocumentsWriterPerThread(writer.newSegmentName(),
directory, config, infoStream, deleteQueue, infos,
- writer.pendingNumDocs);
+ writer.pendingNumDocs, writer.enableTestPoints);
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1646242&r1=1646241&r2=1646242&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Wed Dec 17 13:55:47 2014
@@ -155,9 +155,10 @@ class DocumentsWriterPerThread {
final IntBlockPool.Allocator intBlockAllocator;
private final AtomicLong pendingNumDocs;
private final LiveIndexWriterConfig indexWriterConfig;
+ private final boolean enableTestPoints;
public DocumentsWriterPerThread(String segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue,
- FieldInfos.Builder fieldInfos, AtomicLong pendingNumDocs) throws IOException {
+ FieldInfos.Builder fieldInfos, AtomicLong pendingNumDocs, boolean enableTestPoints) throws IOException {
this.directoryOrig = directory;
this.directory = new TrackingDirectoryWrapper(directory);
this.fieldInfos = fieldInfos;
@@ -184,6 +185,7 @@ class DocumentsWriterPerThread {
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
consumer = indexWriterConfig.getIndexingChain().getChain(this);
+ this.enableTestPoints = enableTestPoints;
}
public FieldInfos.Builder getFieldInfosBuilder() {
@@ -191,7 +193,8 @@ class DocumentsWriterPerThread {
}
final void testPoint(String message) {
- if (infoStream.isEnabled("TP")) {
+ if (enableTestPoints) {
+ assert infoStream.isEnabled("TP"); // don't enable unless you need them.
infoStream.message("TP", message);
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1646242&r1=1646241&r2=1646242&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Wed Dec 17 13:55:47 2014
@@ -214,6 +214,9 @@ public class IndexWriter implements Clos
static int getActualMaxDocs() {
return IndexWriter.actualMaxDocs;
}
+
+ /** Used only for testing. */
+ boolean enableTestPoints = false;
private static final int UNBOUNDED_MAX_MERGE_SEGMENTS = -1;
@@ -4422,7 +4425,8 @@ public class IndexWriter implements Clos
// startMergeInit
// DocumentsWriter.ThreadState.init start
private final void testPoint(String message) {
- if (infoStream.isEnabled("TP")) {
+ if (enableTestPoints) {
+ assert infoStream.isEnabled("TP"); // don't enable unless you need them.
infoStream.message("TP", message);
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1646242&r1=1646241&r2=1646242&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Dec 17 13:55:47 2014
@@ -2350,7 +2350,7 @@ public class TestIndexWriter extends Luc
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
final SetOnce<IndexWriter> iwRef = new SetOnce<>();
- iwc.setInfoStream(new RandomIndexWriter.TestPointInfoStream(iwc.getInfoStream(), new RandomIndexWriter.TestPoint() {
+ IndexWriter evilWriter = RandomIndexWriter.mockIndexWriter(dir, iwc, new RandomIndexWriter.TestPoint() {
@Override
public void apply(String message) {
if ("startCommitMerge".equals(message)) {
@@ -2359,8 +2359,7 @@ public class TestIndexWriter extends Luc
iwRef.get().setKeepFullyDeletedSegments(true);
}
}
- }));
- IndexWriter evilWriter = new IndexWriter(dir, iwc);
+ });
iwRef.set(evilWriter);
for (int i = 0; i < 1000; i++) {
addDoc(evilWriter);
@@ -2568,10 +2567,12 @@ public class TestIndexWriter extends Luc
final CountDownLatch startCommit = new CountDownLatch(1);
final CountDownLatch finishCommit = new CountDownLatch(1);
- // infostream that "takes a long time" to commit
- InfoStream slowCommittingInfoStream = new InfoStream() {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(null);
+ // use an infostream that "takes a long time" to commit
+ final IndexWriter iw = RandomIndexWriter.mockIndexWriter(dir, iwc, new RandomIndexWriter.TestPoint() {
@Override
- public void message(String component, String message) {
+ public void apply(String message) {
if (message.equals("finishStartCommit")) {
startCommit.countDown();
try {
@@ -2581,20 +2582,7 @@ public class TestIndexWriter extends Luc
}
}
}
-
- @Override
- public boolean isEnabled(String component) {
- return true;
- }
-
- @Override
- public void close() throws IOException {}
- };
-
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(null);
- iwc.setInfoStream(slowCommittingInfoStream);
- final IndexWriter iw = new IndexWriter(dir, iwc);
+ });
new Thread() {
@Override
public void run() {
Copied: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java (from r1646240, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java?p2=lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java&r1=1646240&r2=1646242&rev=1646242&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java Wed Dec 17 13:55:47 2014
@@ -57,7 +57,7 @@ public class TestInfoStream extends Luce
public void testTestPointsOn() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(null);
- AtomicBoolean seenTestPoint = new AtomicBoolean();
+ final AtomicBoolean seenTestPoint = new AtomicBoolean();
iwc.setInfoStream(new InfoStream() {
@Override
public void close() throws IOException {}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1646242&r1=1646241&r2=1646242&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java Wed Dec 17 13:55:47 2014
@@ -45,16 +45,6 @@ public class TestStressIndexing2 extends
static int maxBufferedDocs=3;
static int seed=0;
- public final class YieldTestPoint implements RandomIndexWriter.TestPoint {
-
- @Override
- public void apply(String name) {
- // if (name.equals("startCommit")) {
- if (random().nextInt(4) == 2)
- Thread.yield();
- }
- }
-//
public void testRandomIWReader() throws Throwable {
Directory dir = newDirectory();
@@ -148,7 +138,7 @@ public class TestStressIndexing2 extends
.setOpenMode(OpenMode.CREATE)
.setRAMBufferSizeMB(0.1)
.setMaxBufferedDocs(maxBufferedDocs)
- .setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
+ .setMergePolicy(newLogMergePolicy()), random());
w.commit();
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setNoCFSRatio(0.0);
@@ -203,7 +193,7 @@ public class TestStressIndexing2 extends
.setMaxBufferedDocs(maxBufferedDocs)
.setIndexerThreadPool(new DocumentsWriterPerThreadPool(maxThreadStates))
.setReaderPooling(doReaderPooling)
- .setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
+ .setMergePolicy(newLogMergePolicy()), random());
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setNoCFSRatio(0.0);
lmp.setMergeFactor(mergeFactor);
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1646242&r1=1646241&r2=1646242&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Wed Dec 17 13:55:47 2014
@@ -50,7 +50,7 @@ public class RandomIndexWriter implement
private boolean getReaderCalled;
private final Codec codec; // sugar
-
+ /** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */
public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
// Randomly calls Thread.yield so we mixup thread scheduling
final Random random = new Random(r.nextLong());
@@ -63,9 +63,12 @@ public class RandomIndexWriter implement
});
}
+ /** Returns an indexwriter that enables the specified test point */
public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, TestPoint testPoint) throws IOException {
conf.setInfoStream(new TestPointInfoStream(conf.getInfoStream(), testPoint));
- return new IndexWriter(dir, conf);
+ IndexWriter iw = new IndexWriter(dir, conf);
+ iw.enableTestPoints = true;
+ return iw;
}
/** create a RandomIndexWriter with a random config: Uses MockAnalyzer */