You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2015/02/17 20:07:00 UTC
hadoop git commit: HDFS-7795. Show warning if not all favored nodes
were chosen by namenode. Contributed by Kihwal Lee. (cherry picked from
commit db6606223ca2e17aa7e1b2e2be13c1a19d8e7465)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 f52fcdc2e -> 8b37b4a78
HDFS-7795. Show warning if not all favored nodes were chosen by namenode. Contributed by Kihwal Lee.
(cherry picked from commit db6606223ca2e17aa7e1b2e2be13c1a19d8e7465)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b37b4a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b37b4a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b37b4a7
Branch: refs/heads/branch-2
Commit: 8b37b4a78b3eec139e19c8f0ae1c0311c53afe26
Parents: f52fcdc
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Feb 17 13:05:43 2015 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Feb 17 13:06:41 2015 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 26 ++++++++++++++------
2 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b37b4a7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b95eded..9bd4fa8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -348,6 +348,9 @@ Release 2.7.0 - UNRELEASED
HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh)
+ HDFS-7795. Show warning if not all favored nodes were chosen by namenode
+ (kihwal)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b37b4a7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 3ed957b..fc9562f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -35,6 +35,7 @@ import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -1443,7 +1444,7 @@ public class DFSOutputStream extends FSOutputSummer
ExtendedBlock blockCopy = new ExtendedBlock(block);
blockCopy.setNumBytes(blockSize);
- boolean[] targetPinnings = getPinnings(nodes);
+ boolean[] targetPinnings = getPinnings(nodes, true);
// send the request
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
@@ -1537,20 +1538,29 @@ public class DFSOutputStream extends FSOutputSummer
}
}
- private boolean[] getPinnings(DatanodeInfo[] nodes) {
+ private boolean[] getPinnings(DatanodeInfo[] nodes, boolean shouldLog) {
if (favoredNodes == null) {
return null;
} else {
boolean[] pinnings = new boolean[nodes.length];
+ HashSet<String> favoredSet =
+ new HashSet<String>(Arrays.asList(favoredNodes));
for (int i = 0; i < nodes.length; i++) {
- pinnings[i] = false;
- for (int j = 0; j < favoredNodes.length; j++) {
- if (nodes[i].getXferAddrWithHostname().equals(favoredNodes[j])) {
- pinnings[i] = true;
- break;
- }
+ pinnings[i] = favoredSet.remove(nodes[i].getXferAddrWithHostname());
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug(nodes[i].getXferAddrWithHostname() +
+ " was chosen by name node (favored=" + pinnings[i] +
+ ").");
}
}
+ if (shouldLog && !favoredSet.isEmpty()) {
+ // There is one or more favored nodes that were not allocated.
+ DFSClient.LOG.warn(
+ "These favored nodes were specified but not chosen: " +
+ favoredSet +
+ " Specified favored nodes: " + Arrays.toString(favoredNodes));
+
+ }
return pinnings;
}
}