You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2019/12/25 05:58:44 UTC
[hadoop] branch trunk updated: HDFS-12999. When reach the end of
the block group,
it may not need to flush all the data packets(flushAllInternals) twice.
Contributed by lufei and Fei Hui.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new df622cf HDFS-12999. When reach the end of the block group, it may not need to flush all the data packets(flushAllInternals) twice. Contributed by lufei and Fei Hui.
df622cf is described below
commit df622cf4a32ee172ded6c4b3b97a1e49befc4f10
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Wed Dec 25 11:07:25 2019 +0530
HDFS-12999. When reach the end of the block group, it may not need to flush all the data packets(flushAllInternals) twice. Contributed by lufei and Fei Hui.
---
.../org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index ff81995..b1c55a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -572,7 +572,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
// if this is the end of the block group, end each internal block
if (shouldEndBlockGroup()) {
flushAllInternals();
- checkStreamerFailures();
+ checkStreamerFailures(false);
for (int i = 0; i < numAllBlocks; i++) {
final StripedDataStreamer s = setCurrentStreamer(i);
if (s.isHealthy()) {
@@ -583,7 +583,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
}
} else {
// check failure state for all the streamers. Bump GS if necessary
- checkStreamerFailures();
+ checkStreamerFailures(true);
}
}
setCurrentStreamer(next);
@@ -639,15 +639,18 @@ public class DFSStripedOutputStream extends DFSOutputStream
* written a full stripe (i.e., enqueue all packets for a full stripe), or
* when we're closing the outputstream.
*/
- private void checkStreamerFailures() throws IOException {
+ private void checkStreamerFailures(boolean isNeedFlushAllPackets)
+ throws IOException {
Set<StripedDataStreamer> newFailed = checkStreamers();
if (newFailed.size() == 0) {
return;
}
- // for healthy streamers, wait till all of them have fetched the new block
- // and flushed out all the enqueued packets.
- flushAllInternals();
+ if (isNeedFlushAllPackets) {
+ // for healthy streamers, wait till all of them have fetched the new block
+ // and flushed out all the enqueued packets.
+ flushAllInternals();
+ }
// recheck failed streamers again after the flush
newFailed = checkStreamers();
while (newFailed.size() > 0) {
@@ -1204,7 +1207,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
// flush all the data packets
flushAllInternals();
// check failures
- checkStreamerFailures();
+ checkStreamerFailures(false);
for (int i = 0; i < numAllBlocks; i++) {
final StripedDataStreamer s = setCurrentStreamer(i);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org