You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/11/10 15:40:21 UTC
svn commit: r1713649 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/index/ lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/index/
Author: mikemccand
Date: Tue Nov 10 14:40:20 2015
New Revision: 1713649
URL: http://svn.apache.org/viewvc?rev=1713649&view=rev
Log:
LUCENE-6849: expose IndexWriter.flush to move in-memory segments to disk
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1713649&r1=1713648&r2=1713649&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Tue Nov 10 14:40:20 2015
@@ -69,6 +69,10 @@ API Changes
* LUCENE-6884: Analyzer.tokenStream() and Tokenizer.setReader() are no longer
declared as throwing IOException. (Alan Woodward)
+* LUCENE-6849: Expose IndexWriter.flush() method, to move all
+ in-memory segments to disk without opening a near-real-time reader
+ nor calling fsync (Robert Muir, Simon Willnauer, Mike McCandless)
+
Optimizations
* LUCENE-6708: TopFieldCollector does not compute the score several times on the
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1713649&r1=1713648&r2=1713649&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Nov 10 14:40:20 2015
@@ -3058,6 +3058,12 @@ public class IndexWriter implements Clos
return Thread.holdsLock(fullFlushLock);
}
+ /** Moves all in-memory segments to the {@link Directory}, but does not commit
+ * (fsync) them (call {@link #commit} for that). */
+ public final void flush() throws IOException {
+ flush(true, true);
+ }
+
/**
* Flush all in-memory buffered updates (adds and deletes)
* to the Directory.
@@ -3065,8 +3071,7 @@ public class IndexWriter implements Clos
* deletes or docs were flushed) if necessary
* @param applyAllDeletes whether pending deletes should also
*/
- // why protected
- protected final void flush(boolean triggerMerge, boolean applyAllDeletes) throws IOException {
+ final void flush(boolean triggerMerge, boolean applyAllDeletes) throws IOException {
// NOTE: this method cannot be sync'd because
// maybeMerge() in turn calls mergeScheduler.merge which
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1713649&r1=1713648&r2=1713649&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Nov 10 14:40:20 2015
@@ -520,8 +520,9 @@ public class TestIndexWriter extends Luc
customType.setStoreTermVectorPositions(true);
customType.setStoreTermVectorOffsets(true);
doc.add(newField("field", "aaa", customType));
- for(int i=0;i<19;i++)
+ for(int i=0;i<19;i++) {
writer.addDocument(doc);
+ }
writer.flush(false, true);
writer.close();
SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
@@ -531,6 +532,25 @@ public class TestIndexWriter extends Luc
dir.close();
}
+ public void testFlushWithNoCommit() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ writer.commit();
+
+ writer.addDocument(doc);
+ writer.flush();
+ DirectoryReader r = DirectoryReader.open(dir);
+ assertEquals(1, r.maxDoc());
+ writer.commit();
+ DirectoryReader r2 = DirectoryReader.openIfChanged(r);
+ assertNotNull(r2);
+ assertEquals(2, r2.maxDoc());
+ IOUtils.close(r2, r, writer, dir);
+ }
+
// Make sure we can flush segment w/ norms, then add
// empty doc (no norms) and flush
public void testEmptyDocAfterFlushingRealDoc() throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1713649&r1=1713648&r2=1713649&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Tue Nov 10 14:40:20 2015
@@ -24,7 +24,6 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
@@ -49,7 +48,6 @@ public class RandomIndexWriter implement
int flushAt;
private double flushAtFactor = 1.0;
private boolean getReaderCalled;
- private final Codec codec; // sugar
private final Analyzer analyzer; // only if WE created it (then we close it)
/** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */
@@ -120,7 +118,6 @@ public class RandomIndexWriter implement
} else {
analyzer = null;
}
- codec = w.getConfig().getCodec();
if (LuceneTestCase.VERBOSE) {
System.out.println("RIW dir=" + dir);
}
@@ -173,16 +170,23 @@ public class RandomIndexWriter implement
w.addDocument(doc);
}
- maybeCommit();
+ maybeFlushOrCommit();
}
- private void maybeCommit() throws IOException {
+ private void maybeFlushOrCommit() throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
if (docCount++ == flushAt) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("RIW.add/updateDocument: now doing a commit at docCount=" + docCount);
+ if (r.nextBoolean()) {
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("RIW.add/updateDocument: now doing a flush at docCount=" + docCount);
+ }
+ w.flush();
+ } else {
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("RIW.add/updateDocument: now doing a commit at docCount=" + docCount);
+ }
+ w.commit();
}
- w.commit();
flushAt += TestUtil.nextInt(r, (int) (flushAtFactor * 10), (int) (flushAtFactor * 1000));
if (flushAtFactor < 2e6) {
// gradually but exponentially increase time b/w flushes
@@ -194,13 +198,13 @@ public class RandomIndexWriter implement
public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.addDocuments(docs);
- maybeCommit();
+ maybeFlushOrCommit();
}
public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.updateDocuments(delTerm, docs);
- maybeCommit();
+ maybeFlushOrCommit();
}
/**
@@ -241,7 +245,7 @@ public class RandomIndexWriter implement
} else {
w.updateDocument(t, doc);
}
- maybeCommit();
+ maybeFlushOrCommit();
}
public void addIndexes(Directory... dirs) throws IOException {
@@ -438,6 +442,11 @@ public class RandomIndexWriter implement
}
}
+ /** Writes all in-memory segments to the {@link Directory}. */
+ public final void flush() throws IOException {
+ w.flush();
+ }
+
/**
* Simple interface that is executed for each <tt>TP</tt> {@link InfoStream} component
* message. See also {@link RandomIndexWriter#mockIndexWriter(Random, Directory, IndexWriterConfig, TestPoint)}