You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/02/01 09:03:47 UTC
[3/3] hadoop git commit: HDFS-9566. Remove expensive
'BlocksMap#getStorages(Block b,
final DatanodeStorage.State state)' method (Contributed by Daryn Sharp)
HDFS-9566. Remove expensive 'BlocksMap#getStorages(Block b, final DatanodeStorage.State state)' method (Contributed by Daryn Sharp)
(cherry picked from commit e418bd1fb0568ce7ae22f588fea2dd9c95567383)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(cherry picked from commit 71374cca9d0981ac21bb588b85cbc3b0b8908be0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6eae76f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6eae76f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6eae76f7
Branch: refs/heads/branch-2.8
Commit: 6eae76f7a5c3d1a556291e705c360b9baa4fcb7a
Parents: a991605
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Feb 1 13:24:05 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Feb 1 13:32:59 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/blockmanagement/BlockManager.java | 15 ++++++++++++---
.../hdfs/server/blockmanagement/BlocksMap.java | 20 --------------------
3 files changed, 15 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eae76f7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d0813c4..a1c5794 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1684,6 +1684,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9682. Fix a typo "aplication" in HttpFS document.
(Weiwei Yang via aajisaka)
+ HDFS-9566. Remove expensive 'BlocksMap#getStorages(Block b, final
+ DatanodeStorage.State state)' method (Daryn Sharp via vinayakumarb)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eae76f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index b477fad..e6ec759 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1137,7 +1137,10 @@ public class BlockManager implements BlockStatsMXBean {
return;
}
StringBuilder datanodes = new StringBuilder();
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
+ for (DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ if (storage.getState() != State.NORMAL) {
+ continue;
+ }
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
invalidateBlocks.add(b, node, false);
datanodes.append(node).append(" ");
@@ -2912,7 +2915,10 @@ public class BlockManager implements BlockStatsMXBean {
Collection<DatanodeStorageInfo> nonExcess = new ArrayList<DatanodeStorageInfo>();
Collection<DatanodeDescriptor> corruptNodes = corruptReplicas
.getNodes(block);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(block, State.NORMAL)) {
+ for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
+ if (storage.getState() != State.NORMAL) {
+ continue;
+ }
final DatanodeDescriptor cur = storage.getDatanodeDescriptor();
if (storage.areBlockContentsStale()) {
LOG.trace("BLOCK* processOverReplicatedBlock: Postponing {}"
@@ -3294,7 +3300,10 @@ public class BlockManager implements BlockStatsMXBean {
// else proceed with fast case
int live = 0;
Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
+ for (DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ if (storage.getState() != State.NORMAL) {
+ continue;
+ }
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
if ((nodesCorrupt == null) || (!nodesCorrupt.contains(node)))
live++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eae76f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index cbf78cf..72a45c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -21,13 +21,9 @@ import java.util.Iterator;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-
/**
* This class maintains the map from a block to its metadata.
* block's metadata currently includes blockCollection it belongs to and
@@ -141,22 +137,6 @@ class BlocksMap {
}
/**
- * Searches for the block in the BlocksMap and
- * returns {@link Iterable} of the storages the block belongs to
- * <i>that are of the given {@link DatanodeStorage.State state}</i>.
- *
- * @param state DatanodeStorage state by which to filter the returned Iterable
- */
- Iterable<DatanodeStorageInfo> getStorages(Block b, final DatanodeStorage.State state) {
- return Iterables.filter(getStorages(blocks.get(b)), new Predicate<DatanodeStorageInfo>() {
- @Override
- public boolean apply(DatanodeStorageInfo storage) {
- return storage.getState() == state;
- }
- });
- }
-
- /**
* For a block that has already been retrieved from the BlocksMap
* returns {@link Iterable} of the storages the block belongs to.
*/