You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/02/01 19:41:04 UTC
[45/50] [abbrv] hadoop git commit: HDFS-9566. Remove expensive
'BlocksMap#getStorages(Block b,
final DatanodeStorage.State state)' method (Contributed by Daryn Sharp)
HDFS-9566. Remove expensive 'BlocksMap#getStorages(Block b, final DatanodeStorage.State state)' method (Contributed by Daryn Sharp)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e418bd1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e418bd1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e418bd1f
Branch: refs/heads/HDFS-7240
Commit: e418bd1fb0568ce7ae22f588fea2dd9c95567383
Parents: 2673cba
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Feb 1 13:24:05 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Feb 1 13:24:05 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/blockmanagement/BlockManager.java | 16 ++++++++++++----
.../hdfs/server/blockmanagement/BlocksMap.java | 20 --------------------
3 files changed, 15 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e418bd1f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f7487fe..432e686 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2655,6 +2655,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9682. Fix a typo "aplication" in HttpFS document.
(Weiwei Yang via aajisaka)
+ HDFS-9566. Remove expensive 'BlocksMap#getStorages(Block b, final
+ DatanodeStorage.State state)' method (Daryn Sharp via vinayakumarb)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e418bd1f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index a76429e..587e6b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1212,8 +1212,10 @@ public class BlockManager implements BlockStatsMXBean {
return;
}
StringBuilder datanodes = new StringBuilder();
- for(DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock,
- State.NORMAL)) {
+ for (DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock)) {
+ if (storage.getState() != State.NORMAL) {
+ continue;
+ }
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
final Block b = getBlockOnStorage(storedBlock, storage);
if (b != null) {
@@ -3164,7 +3166,10 @@ public class BlockManager implements BlockStatsMXBean {
Collection<DatanodeStorageInfo> nonExcess = new ArrayList<>();
Collection<DatanodeDescriptor> corruptNodes = corruptReplicas
.getNodes(block);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(block, State.NORMAL)) {
+ for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
+ if (storage.getState() != State.NORMAL) {
+ continue;
+ }
final DatanodeDescriptor cur = storage.getDatanodeDescriptor();
if (storage.areBlockContentsStale()) {
LOG.trace("BLOCK* processOverReplicatedBlock: Postponing {}"
@@ -3665,7 +3670,10 @@ public class BlockManager implements BlockStatsMXBean {
// else proceed with fast case
int live = 0;
Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
+ for (DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
+ if (storage.getState() != State.NORMAL) {
+ continue;
+ }
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
if ((nodesCorrupt == null) || (!nodesCorrupt.contains(node)))
live++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e418bd1f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index ed05e3a..47a21fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -21,13 +21,9 @@ import java.util.Iterator;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-
/**
* This class maintains the map from a block to its metadata.
* block's metadata currently includes blockCollection it belongs to and
@@ -150,22 +146,6 @@ class BlocksMap {
}
/**
- * Searches for the block in the BlocksMap and
- * returns {@link Iterable} of the storages the block belongs to
- * <i>that are of the given {@link DatanodeStorage.State state}</i>.
- *
- * @param state DatanodeStorage state by which to filter the returned Iterable
- */
- Iterable<DatanodeStorageInfo> getStorages(Block b, final DatanodeStorage.State state) {
- return Iterables.filter(getStorages(blocks.get(b)), new Predicate<DatanodeStorageInfo>() {
- @Override
- public boolean apply(DatanodeStorageInfo storage) {
- return storage.getState() == state;
- }
- });
- }
-
- /**
* For a block that has already been retrieved from the BlocksMap
* returns {@link Iterable} of the storages the block belongs to.
*/