You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/02/12 19:59:39 UTC

hadoop git commit: HDFS-9794. Streamer threads may leak if failure happens when closing the striped outputstream. Contributed by Jing Zhao.

Repository: hadoop
Updated Branches:
  refs/heads/trunk b21bbe9ed -> f3c91a41a


HDFS-9794. Streamer threads may leak if failure happens when closing the striped outputstream. Contributed by Jing Zhao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3c91a41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3c91a41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3c91a41

Branch: refs/heads/trunk
Commit: f3c91a41a5bf16542eca7f09787eb1727fd18e08
Parents: b21bbe9
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Feb 12 10:59:31 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Feb 12 10:59:31 2016 -0800

----------------------------------------------------------------------
 .../hadoop/hdfs/DFSStripedOutputStream.java     | 63 +++++++++++---------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +
 2 files changed, 38 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c91a41/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 8292d0a..3dfdbee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -822,7 +822,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
           streamer.closeSocket();
         } catch (Exception e) {
           try {
-            handleCurrentStreamerFailure("force=" + force, e);
+            handleStreamerFailure("force=" + force, e, streamer);
           } catch (IOException ioe) {
             b.add(ioe);
           }
@@ -947,38 +947,45 @@ public class DFSStripedOutputStream extends DFSOutputStream {
     }
 
     try {
-      // flush from all upper layers
-      flushBuffer();
-      // if the last stripe is incomplete, generate and write parity cells
-      if (generateParityCellsForLastStripe()) {
-        writeParityCells();
-      }
-      enqueueAllCurrentPackets();
-
-      // flush all the data packets
-      flushAllInternals();
-      // check failures
-      checkStreamerFailures();
-
-      for (int i = 0; i < numAllBlocks; i++) {
-        final StripedDataStreamer s = setCurrentStreamer(i);
-        if (s.isHealthy()) {
-          try {
-            if (s.getBytesCurBlock() > 0) {
-              setCurrentPacketToEmpty();
+      try {
+        // flush from all upper layers
+        flushBuffer();
+        // if the last stripe is incomplete, generate and write parity cells
+        if (generateParityCellsForLastStripe()) {
+          writeParityCells();
+        }
+        enqueueAllCurrentPackets();
+
+        // flush all the data packets
+        flushAllInternals();
+        // check failures
+        checkStreamerFailures();
+
+        for (int i = 0; i < numAllBlocks; i++) {
+          final StripedDataStreamer s = setCurrentStreamer(i);
+          if (s.isHealthy()) {
+            try {
+              if (s.getBytesCurBlock() > 0) {
+                setCurrentPacketToEmpty();
+              }
+              // flush the last "close" packet to Datanode
+              flushInternal();
+            } catch (Exception e) {
+              // TODO for both close and endBlock, we currently do not handle
+              // failures when sending the last packet. We actually do not need to
+              // bump GS for this kind of failure. Thus counting the total number
+              // of failures may be good enough.
             }
-            // flush the last "close" packet to Datanode
-            flushInternal();
-          } catch(Exception e) {
-            // TODO for both close and endBlock, we currently do not handle
-            // failures when sending the last packet. We actually do not need to
-            // bump GS for this kind of failure. Thus counting the total number
-            // of failures may be good enough.
           }
         }
+      } finally {
+        // Failures may happen when flushing data/parity data out. Exceptions
+        // may be thrown if more than 3 streamers fail, or updatePipeline RPC
+        // fails. Streamers may keep waiting for the new block/GS information.
+        // Thus need to force closing these threads.
+        closeThreads(true);
       }
 
-      closeThreads(false);
       try (TraceScope ignored =
                dfsClient.getTracer().newScope("completeFile")) {
         completeFile(currentBlockGroup);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c91a41/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8633c31..687e35f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -426,6 +426,9 @@ Trunk (Unreleased)
     HDFS-9789. Correctly update DataNode's scheduled block size when writing
     small EC file. (jing9)
 
+    HDFS-9794. Streamer threads may leak if failure happens when closing the
+    striped outputstream. (jing9)
+
     BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
 
       HDFS-7347. Configurable erasure coding policy for individual files and