You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2018/08/30 09:28:44 UTC
[2/2] lucene-solr:branch_7x: LUCENE-8471: Add
IndexWriter.getFlushingBytes() method
LUCENE-8471: Add IndexWriter.getFlushingBytes() method
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/06d6e63f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/06d6e63f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/06d6e63f
Branch: refs/heads/branch_7x
Commit: 06d6e63fdac888578545834116be6842fc12f65e
Parents: bf7d107
Author: Alan Woodward <ro...@apache.org>
Authored: Thu Aug 30 09:56:09 2018 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Thu Aug 30 10:12:00 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../java/org/apache/lucene/index/DocumentsWriter.java | 7 +++++++
.../lucene/index/DocumentsWriterFlushControl.java | 8 ++++----
.../src/java/org/apache/lucene/index/IndexWriter.java | 8 ++++++++
.../lucene/index/TestFlushByRamOrCountsPolicy.java | 14 +++++++-------
5 files changed, 29 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index bc7aeda..541a754 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -55,6 +55,9 @@ API Changes:
word-by-word. The UH's public internal APIs changed a bit in the process.
(David Smiley)
+* LUCENE-8471: IndexWriter.getFlushingBytes() returns how many bytes are currently
+ being flushed to disk. (Alan Woodward)
+
Bug Fixes:
* LUCENE-8445: Tighten condition when two planes are identical to prevent constructing
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
index cbbf22e..136bd81 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
@@ -762,4 +762,11 @@ final class DocumentsWriter implements Closeable, Accountable {
public long ramBytesUsed() {
return flushControl.ramBytesUsed();
}
+
+ /**
+ * Returns the number of bytes currently being flushed
+ */
+ public long getFlushingBytes() {
+ return flushControl.getFlushingBytes();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
index ad5b7e4..5b6f8af 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
@@ -47,7 +47,7 @@ final class DocumentsWriterFlushControl implements Accountable {
private final long hardMaxBytesPerDWPT;
private long activeBytes = 0;
- private long flushBytes = 0;
+ private volatile long flushBytes = 0;
private volatile int numPending = 0;
private int numDocsSinceStalled = 0; // only with assert
final AtomicBoolean flushDeletes = new AtomicBoolean(false);
@@ -86,7 +86,7 @@ final class DocumentsWriterFlushControl implements Accountable {
return activeBytes;
}
- public synchronized long flushBytes() {
+ public long getFlushingBytes() {
return flushBytes;
}
@@ -257,11 +257,11 @@ final class DocumentsWriterFlushControl implements Accountable {
if (stall != stallControl.anyStalledThreads()) {
if (stall) {
infoStream.message("DW", String.format(Locale.ROOT, "now stalling flushes: netBytes: %.1f MB flushBytes: %.1f MB fullFlush: %b",
- netBytes()/1024./1024., flushBytes()/1024./1024., fullFlush));
+ netBytes()/1024./1024., getFlushingBytes()/1024./1024., fullFlush));
stallStartNS = System.nanoTime();
} else {
infoStream.message("DW", String.format(Locale.ROOT, "done stalling flushes for %.1f msec: netBytes: %.1f MB flushBytes: %.1f MB fullFlush: %b",
- (System.nanoTime()-stallStartNS)/1000000., netBytes()/1024./1024., flushBytes()/1024./1024., fullFlush));
+ (System.nanoTime()-stallStartNS)/1000000., netBytes()/1024./1024., getFlushingBytes()/1024./1024., fullFlush));
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index fbc9f37..0ce5882 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -570,6 +570,14 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
return docWriter.ramBytesUsed();
}
+ /**
+ * Returns the number of bytes currently being flushed
+ */
+ public final long getFlushingBytes() {
+ ensureOpen();
+ return docWriter.getFlushingBytes();
+ }
+
final long getReaderPoolRamBytesUsed() {
return readerPool.ramBytesUsed();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/06d6e63f/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
index ca5aba8..fa6cfa9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
@@ -81,7 +81,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
DocumentsWriter docsWriter = writer.getDocsWriter();
assertNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
- assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes());
+ assertEquals(" bytes must be 0 after init", 0, writer.getFlushingBytes());
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.length; x++) {
@@ -95,7 +95,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
}
final long maxRAMBytes = (long) (iwc.getRAMBufferSizeMB() * 1024. * 1024.);
assertEquals(" all flushes must be due numThreads=" + numThreads, 0,
- flushControl.flushBytes());
+ writer.getFlushingBytes());
assertEquals(numDocumentsToIndex, writer.numDocs());
assertEquals(numDocumentsToIndex, writer.maxDoc());
assertTrue("peak bytes without flush exceeded watermark",
@@ -136,7 +136,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
DocumentsWriter docsWriter = writer.getDocsWriter();
assertNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
- assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes());
+ assertEquals(" bytes must be 0 after init", 0, writer.getFlushingBytes());
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.length; x++) {
@@ -150,7 +150,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
}
assertEquals(" all flushes must be due numThreads=" + numThreads[i], 0,
- flushControl.flushBytes());
+ writer.getFlushingBytes());
assertEquals(numDocumentsToIndex, writer.numDocs());
assertEquals(numDocumentsToIndex, writer.maxDoc());
assertTrue("peak bytes without flush exceeded watermark",
@@ -182,7 +182,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
assertNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
- assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes());
+ assertEquals(" bytes must be 0 after init", 0, writer.getFlushingBytes());
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.length; x++) {
@@ -194,7 +194,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
for (int x = 0; x < threads.length; x++) {
threads[x].join();
}
- assertEquals(" all flushes must be due", 0, flushControl.flushBytes());
+ assertEquals(" all flushes must be due", 0, writer.getFlushingBytes());
assertEquals(numDocumentsToIndex, writer.numDocs());
assertEquals(numDocumentsToIndex, writer.maxDoc());
if (flushPolicy.flushOnRAM() && !flushPolicy.flushOnDocCount()) {
@@ -255,7 +255,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
DocumentsWriter docsWriter = writer.getDocsWriter();
assertNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
- assertEquals(" all flushes must be due", 0, flushControl.flushBytes());
+ assertEquals(" all flushes must be due", 0, writer.getFlushingBytes());
assertEquals(numDocumentsToIndex, writer.numDocs());
assertEquals(numDocumentsToIndex, writer.maxDoc());
if (numThreads[i] == 1) {