You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/09/30 17:42:36 UTC
[36/50] [abbrv] hadoop git commit: HDFS-8920. Erasure Coding: when
recovering lost blocks,
logs can be too verbose and hurt performance. Contributed by Rui Li
HDFS-8920. Erasure Coding: when recovering lost blocks, logs can be too verbose and hurt performance. Contributed by Rui Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bff8ca1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bff8ca1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bff8ca1
Branch: refs/heads/trunk
Commit: 7bff8ca1c872ea534a96cbbc5f70134574e289ce
Parents: 1080c37
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Sep 23 14:13:26 2015 +0800
Committer: Zhe Zhang <zh...@cloudera.com>
Committed: Tue Sep 22 13:27:24 2015 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 ++
.../org/apache/hadoop/hdfs/DFSInputStream.java | 15 +++++++--
.../hadoop/hdfs/DFSStripedInputStream.java | 34 ++++++++++++++++++++
3 files changed, 49 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bff8ca1/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 0e21d22..b79ce64 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -444,3 +444,6 @@
HDFS-9091. Erasure Coding: Provide DistributedFilesystem API to
getAllErasureCodingPolicies. (Rakesh R via zhz)
+
+ HDFS-8920. Erasure Coding: when recovering lost blocks, logs can be too
+ verbose and hurt performance. (Rui Li via Kai Zheng)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bff8ca1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index a5911cc..385ba4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -1057,9 +1057,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
}
if (chosenNode == null) {
- DFSClient.LOG.warn("No live nodes contain block " + block.getBlock() +
- " after checking nodes = " + Arrays.toString(nodes) +
- ", ignoredNodes = " + ignoredNodes);
+ reportLostBlock(block, ignoredNodes);
return null;
}
final String dnAddr =
@@ -1071,6 +1069,17 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
return new DNAddrPair(chosenNode, targetAddr, storageType);
}
+ /**
+ * Warn the user of a lost block
+ */
+ protected void reportLostBlock(LocatedBlock lostBlock,
+ Collection<DatanodeInfo> ignoredNodes) {
+ DatanodeInfo[] nodes = lostBlock.getLocations();
+ DFSClient.LOG.warn("No live nodes contain block " + lostBlock.getBlock() +
+ " after checking nodes = " + Arrays.toString(nodes) +
+ ", ignoredNodes = " + ignoredNodes);
+ }
+
private static String getBestNodeDNAddrPairErrorString(
DatanodeInfo nodes[], AbstractMap<DatanodeInfo,
DatanodeInfo> deadNodes, Collection<DatanodeInfo> ignoredNodes) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bff8ca1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index b7c22c4..131a552 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -45,8 +45,11 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.EnumSet;
+import java.util.List;
import java.util.Set;
import java.util.Collection;
import java.util.Map;
@@ -154,6 +157,17 @@ public class DFSStripedInputStream extends DFSInputStream {
private StripeRange curStripeRange;
private final CompletionService<Void> readingService;
+ /**
+ * When warning the user of a lost block in striping mode, we remember the
+ * dead nodes we've logged. All other striping blocks on these nodes can be
+ * considered lost too, and we don't want to log a warning for each of them.
+ * This is to prevent the log from being too verbose. Refer to HDFS-8920.
+ *
+ * To minimize the overhead, we only store the datanodeUuid in this set
+ */
+ private final Set<String> warnedNodes = Collections.newSetFromMap(
+ new ConcurrentHashMap<String, Boolean>());
+
DFSStripedInputStream(DFSClient dfsClient, String src,
boolean verifyChecksum, ErasureCodingPolicy ecPolicy,
LocatedBlocks locatedBlocks) throws IOException {
@@ -527,6 +541,26 @@ public class DFSStripedInputStream extends DFSInputStream {
}
}
+ @Override
+ protected void reportLostBlock(LocatedBlock lostBlock,
+ Collection<DatanodeInfo> ignoredNodes) {
+ DatanodeInfo[] nodes = lostBlock.getLocations();
+ if (nodes != null && nodes.length > 0) {
+ List<String> dnUUIDs = new ArrayList<>();
+ for (DatanodeInfo node : nodes) {
+ dnUUIDs.add(node.getDatanodeUuid());
+ }
+ if (!warnedNodes.containsAll(dnUUIDs)) {
+ DFSClient.LOG.warn(Arrays.toString(nodes) + " are unavailable and " +
+ "all striping blocks on them are lost. " +
+ "IgnoredNodes = " + ignoredNodes);
+ warnedNodes.addAll(dnUUIDs);
+ }
+ } else {
+ super.reportLostBlock(lostBlock, ignoredNodes);
+ }
+ }
+
/**
* The reader for reading a complete {@link AlignedStripe}. Note that an
* {@link AlignedStripe} may cross multiple stripes with cellSize width.