You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/07/22 01:14:09 UTC
[25/44] hadoop git commit: HDFS-8794. Improve
CorruptReplicasMap#corruptReplicasMap. (yliu)
HDFS-8794. Improve CorruptReplicasMap#corruptReplicasMap. (yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6d58606
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6d58606
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6d58606
Branch: refs/heads/YARN-1197
Commit: d6d58606b8adf94b208aed5fc2d054b9dd081db1
Parents: ed01dc7
Author: yliu <yl...@apache.org>
Authored: Tue Jul 21 09:20:22 2015 +0800
Committer: yliu <yl...@apache.org>
Committed: Tue Jul 21 09:20:22 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../blockmanagement/CorruptReplicasMap.java | 19 ++++++++++++++-----
.../blockmanagement/TestCorruptReplicaInfo.java | 12 ++++++------
3 files changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cd32c0e..388b553 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -727,6 +727,8 @@ Release 2.8.0 - UNRELEASED
HDFS-7314. When the DFSClient lease cannot be renewed, abort open-for-write
files rather than the entire DFSClient. (mingma)
+ HDFS-8794. Improve CorruptReplicasMap#corruptReplicasMap. (yliu)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
index fc2e234..f83cbaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
@@ -17,12 +17,19 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.Server;
-import java.util.*;
+import com.google.common.annotations.VisibleForTesting;
/**
* Stores information about all corrupt blocks in the File System.
@@ -46,8 +53,8 @@ public class CorruptReplicasMap{
CORRUPTION_REPORTED // client or datanode reported the corruption
}
- private final SortedMap<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap =
- new TreeMap<Block, Map<DatanodeDescriptor, Reason>>();
+ private final Map<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap =
+ new HashMap<Block, Map<DatanodeDescriptor, Reason>>();
/**
* Mark the block belonging to datanode as corrupt.
@@ -181,13 +188,15 @@ public class CorruptReplicasMap{
* @return Up to numExpectedBlocks blocks from startingBlockId if it exists
*
*/
- long[] getCorruptReplicaBlockIds(int numExpectedBlocks,
+ @VisibleForTesting
+ long[] getCorruptReplicaBlockIdsForTesting(int numExpectedBlocks,
Long startingBlockId) {
if (numExpectedBlocks < 0 || numExpectedBlocks > 100) {
return null;
}
- Iterator<Block> blockIt = corruptReplicasMap.keySet().iterator();
+ Iterator<Block> blockIt =
+ new TreeMap<>(corruptReplicasMap).keySet().iterator();
// if the starting block id was specified, iterate over keys until
// we find the matching block. If we find a matching block, break
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6d58606/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
index 21fb54e..4bdaaac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
@@ -73,9 +73,9 @@ public class TestCorruptReplicaInfo {
// Make sure initial values are returned correctly
assertEquals("Number of corrupt blocks must initially be 0", 0, crm.size());
- assertNull("Param n cannot be less than 0", crm.getCorruptReplicaBlockIds(-1, null));
- assertNull("Param n cannot be greater than 100", crm.getCorruptReplicaBlockIds(101, null));
- long[] l = crm.getCorruptReplicaBlockIds(0, null);
+ assertNull("Param n cannot be less than 0", crm.getCorruptReplicaBlockIdsForTesting(-1, null));
+ assertNull("Param n cannot be greater than 100", crm.getCorruptReplicaBlockIdsForTesting(101, null));
+ long[] l = crm.getCorruptReplicaBlockIdsForTesting(0, null);
assertNotNull("n = 0 must return non-null", l);
assertEquals("n = 0 must return an empty list", 0, l.length);
@@ -118,14 +118,14 @@ public class TestCorruptReplicaInfo {
assertTrue("First five block ids not returned correctly ",
Arrays.equals(new long[]{0,1,2,3,4},
- crm.getCorruptReplicaBlockIds(5, null)));
+ crm.getCorruptReplicaBlockIdsForTesting(5, null)));
- LOG.info(crm.getCorruptReplicaBlockIds(10, 7L));
+ LOG.info(crm.getCorruptReplicaBlockIdsForTesting(10, 7L));
LOG.info(block_ids.subList(7, 18));
assertTrue("10 blocks after 7 not returned correctly ",
Arrays.equals(new long[]{8,9,10,11,12,13,14,15,16,17},
- crm.getCorruptReplicaBlockIds(10, 7L)));
+ crm.getCorruptReplicaBlockIdsForTesting(10, 7L)));
}