You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/01/12 19:19:11 UTC
[29/34] hadoop git commit: HDFS-7579. Improve log reporting during
block report rpc failure. Contributed by Charles Lamb.
HDFS-7579. Improve log reporting during block report rpc failure. Contributed by Charles Lamb.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22af33bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22af33bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22af33bf
Branch: refs/heads/HDFS-EC
Commit: 22af33bfb42b50e4294c37540f09dd74e5a4f502
Parents: 786108d
Author: cnauroth <cn...@apache.org>
Authored: Thu Jan 8 15:12:56 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Mon Jan 12 10:18:02 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/datanode/BPServiceActor.java | 76 ++++++++++++--------
.../hdfs/server/protocol/ServerCommand.java | 8 +++
3 files changed, 57 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22af33bf/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9c1be4a..1358edc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -656,6 +656,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs.
(Zhanwei Wang via cnauroth)
+ HDFS-7579. Improve log reporting during block report rpc failure.
+ (Charles Lamb via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22af33bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index d94d056..e6409ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -22,7 +22,10 @@ import static org.apache.hadoop.util.Time.now;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
import com.google.common.base.Joiner;
import org.apache.commons.logging.Log;
@@ -458,7 +461,7 @@ class BPServiceActor implements Runnable {
return null;
}
- ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
+ final ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
// Flush any block information that precedes the block report. Otherwise
// we have a chance that we will miss the delHint information
@@ -485,40 +488,54 @@ class BPServiceActor implements Runnable {
}
// Send the reports to the NN.
- int numReportsSent;
+ int numReportsSent = 0;
+ int numRPCs = 0;
+ boolean success = false;
long brSendStartTime = now();
- if (totalBlockCount < dnConf.blockReportSplitThreshold) {
- // Below split threshold, send all reports in a single message.
- numReportsSent = 1;
- DatanodeCommand cmd =
- bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports);
- if (cmd != null) {
- cmds.add(cmd);
- }
- } else {
- // Send one block report per message.
- numReportsSent = i;
- for (StorageBlockReport report : reports) {
- StorageBlockReport singleReport[] = { report };
+ try {
+ if (totalBlockCount < dnConf.blockReportSplitThreshold) {
+ // Below split threshold, send all reports in a single message.
DatanodeCommand cmd = bpNamenode.blockReport(
- bpRegistration, bpos.getBlockPoolId(), singleReport);
+ bpRegistration, bpos.getBlockPoolId(), reports);
+ numRPCs = 1;
+ numReportsSent = reports.length;
if (cmd != null) {
cmds.add(cmd);
}
+ } else {
+ // Send one block report per message.
+ for (StorageBlockReport report : reports) {
+ StorageBlockReport singleReport[] = { report };
+ DatanodeCommand cmd = bpNamenode.blockReport(
+ bpRegistration, bpos.getBlockPoolId(), singleReport);
+ numReportsSent++;
+ numRPCs++;
+ if (cmd != null) {
+ cmds.add(cmd);
+ }
+ }
}
+ success = true;
+ } finally {
+ // Log the block report processing stats from Datanode perspective
+ long brSendCost = now() - brSendStartTime;
+ long brCreateCost = brSendStartTime - brCreateStartTime;
+ dn.getMetrics().addBlockReport(brSendCost);
+ final int nCmds = cmds.size();
+ LOG.info((success ? "S" : "Uns") +
+ "uccessfully sent " + numReportsSent +
+ " of " + reports.length +
+ " blockreports for " + totalBlockCount +
+ " total blocks using " + numRPCs +
+ " RPCs. This took " + brCreateCost +
+ " msec to generate and " + brSendCost +
+ " msecs for RPC and NN processing." +
+ " Got back " +
+ ((nCmds == 0) ? "no commands" :
+ ((nCmds == 1) ? "one command: " + cmds.get(0) :
+ (nCmds + " commands: " + Joiner.on("; ").join(cmds)))) +
+ ".");
}
-
- // Log the block report processing stats from Datanode perspective
- long brSendCost = now() - brSendStartTime;
- long brCreateCost = brSendStartTime - brCreateStartTime;
- dn.getMetrics().addBlockReport(brSendCost);
- LOG.info("Sent " + numReportsSent + " blockreports " + totalBlockCount +
- " blocks total. Took " + brCreateCost +
- " msec to generate and " + brSendCost +
- " msecs for RPC and NN processing. " +
- " Got back commands " +
- (cmds.size() == 0 ? "none" : Joiner.on("; ").join(cmds)));
-
scheduleNextBlockReport(startTime);
return cmds.size() == 0 ? null : cmds;
}
@@ -968,7 +985,6 @@ class BPServiceActor implements Runnable {
/**
* Add pending incremental block report for a single block.
- * @param blockID
* @param blockInfo
*/
void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22af33bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java
index 8d7544d..eed9a6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java
@@ -52,4 +52,12 @@ public abstract class ServerCommand {
public int getAction() {
return this.action;
}
+
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ sb.append(getClass().getSimpleName());
+ sb.append("/");
+ sb.append(action);
+ return sb.toString();
+ }
}