You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2016/02/03 23:31:18 UTC
hadoop git commit: HDFS-9601. NNThroughputBenchmark.BlockReportStats
should handle NotReplicatedYetException on adding block (iwasakims)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 a79f570b6 -> 5fd4bd1d8
HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle NotReplicatedYetException on adding block (iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fd4bd1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fd4bd1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fd4bd1d
Branch: refs/heads/branch-2.8
Commit: 5fd4bd1d8c6e02fdca5b1e620cf7416c5e07c6c2
Parents: a79f570
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Fri Jan 22 12:28:38 2016 +0900
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Wed Feb 3 14:29:06 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/namenode/NNThroughputBenchmark.java | 34 +++++++++++++++++++-
2 files changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fd4bd1d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 604e51d..4d41dae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1713,6 +1713,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9748. Avoid duplication in pendingReplications when
addExpectedReplicasToPending is called twice. (Walter Su via jing9)
+ HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle
+ NotReplicatedYetException on adding block (iwasakims)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fd4bd1d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 8a594ed..42cb72f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Groups;
@@ -1181,7 +1182,7 @@ public class NNThroughputBenchmark implements Tool {
throws IOException {
ExtendedBlock prevBlock = null;
for(int jdx = 0; jdx < blocksPerFile; jdx++) {
- LocatedBlock loc = clientProto.addBlock(fileName, clientName,
+ LocatedBlock loc = addBlock(fileName, clientName,
prevBlock, null, HdfsConstants.GRANDFATHER_INODE_ID, null);
prevBlock = loc.getBlock();
for(DatanodeInfo dnInfo : loc.getLocations()) {
@@ -1195,11 +1196,42 @@ public class NNThroughputBenchmark implements Tool {
dataNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration,
bpid, report);
}
+ // IBRs are asynchronously processed by NameNode. The next
+ // ClientProtocol#addBlock() may throw NotReplicatedYetException.
}
return prevBlock;
}
/**
+ * Retry ClientProtocol.addBlock() if it throws NotReplicatedYetException.
+ * Because addBlock() also commits the previous block,
+ * it fails if enough IBRs are not processed by NameNode.
+ */
+ private LocatedBlock addBlock(String src, String clientName,
+ ExtendedBlock previous, DatanodeInfo[] excludeNodes, long fileId,
+ String[] favoredNodes) throws IOException {
+ for (int i = 0; i < 30; i++) {
+ try {
+ return clientProto.addBlock(src, clientName,
+ previous, excludeNodes, fileId, favoredNodes);
+ } catch (NotReplicatedYetException|RemoteException e) {
+ if (e instanceof RemoteException) {
+ String className = ((RemoteException) e).getClassName();
+ if (!className.equals(NotReplicatedYetException.class.getName())) {
+ throw e;
+ }
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException ie) {
+ LOG.warn("interrupted while retrying addBlock.", ie);
+ }
+ }
+ }
+ throw new IOException("failed to add block.");
+ }
+
+ /**
* Does not require the argument
*/
@Override