You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/02/01 19:21:45 UTC
hadoop git commit: HDFS-11370. Optimize NamenodeFsck#getReplicaInfo.
Contributed Takanobu Asanuma.
Repository: hadoop
Updated Branches:
refs/heads/trunk 59c5f1878 -> b6f290d5b
HDFS-11370. Optimize NamenodeFsck#getReplicaInfo. Contributed Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6f290d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6f290d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6f290d5
Branch: refs/heads/trunk
Commit: b6f290d5b660ad157c7076767c619d02b3d0f894
Parents: 59c5f18
Author: Jing Zhao <ji...@apache.org>
Authored: Wed Feb 1 11:21:35 2017 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Wed Feb 1 11:21:35 2017 -0800
----------------------------------------------------------------------
.../BlockUnderConstructionFeature.java | 25 +++++++++++++++++
.../hdfs/server/namenode/NamenodeFsck.java | 28 +++++++++++---------
2 files changed, 41 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6f290d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
index b935f43..7453184 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
@@ -24,7 +24,9 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import java.util.NoSuchElementException;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.COMPLETE;
@@ -110,6 +112,29 @@ public class BlockUnderConstructionFeature {
}
/**
+ * Note that this iterator doesn't guarantee thread-safe. It depends on
+ * external mechanisms such as the FSNamesystem lock for protection.
+ */
+ public Iterator<DatanodeStorageInfo> getExpectedStorageLocationsIterator() {
+ return new Iterator<DatanodeStorageInfo>() {
+ private int index = 0;
+
+ @Override
+ public boolean hasNext() {
+ return index < replicas.length;
+ }
+
+ @Override
+ public DatanodeStorageInfo next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return replicas[index++].getExpectedStorageLocation();
+ }
+ };
+ }
+
+ /**
* @return the index array indicating the block index in each storage. Used
* only by striped blocks.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6f290d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 859f541..7e91935 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -589,23 +589,27 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
return "";
}
final boolean isComplete = storedBlock.isComplete();
- DatanodeStorageInfo[] storages = isComplete ?
- blockManager.getStorages(storedBlock) :
- storedBlock.getUnderConstructionFeature().getExpectedStorageLocations();
+ Iterator<DatanodeStorageInfo> storagesItr;
StringBuilder sb = new StringBuilder(" [");
final boolean isStriped = storedBlock.isStriped();
Map<DatanodeStorageInfo, Long> storage2Id = new HashMap<>();
- if (isStriped && isComplete) {
- long blockId = storedBlock.getBlockId();
- Iterable<StorageAndBlockIndex> sis =
- ((BlockInfoStriped)storedBlock).getStorageAndIndexInfos();
- for (StorageAndBlockIndex si: sis){
- storage2Id.put(si.getStorage(), blockId + si.getBlockIndex());
+ if (isComplete) {
+ if (isStriped) {
+ long blockId = storedBlock.getBlockId();
+ Iterable<StorageAndBlockIndex> sis =
+ ((BlockInfoStriped) storedBlock).getStorageAndIndexInfos();
+ for (StorageAndBlockIndex si : sis) {
+ storage2Id.put(si.getStorage(), blockId + si.getBlockIndex());
+ }
}
+ storagesItr = storedBlock.getStorageInfos();
+ } else {
+ storagesItr = storedBlock.getUnderConstructionFeature()
+ .getExpectedStorageLocationsIterator();
}
- for (int i = 0; i < storages.length; i++) {
- DatanodeStorageInfo storage = storages[i];
+ while (storagesItr.hasNext()) {
+ DatanodeStorageInfo storage = storagesItr.next();
if (isStriped && isComplete) {
long index = storage2Id.get(storage);
sb.append("blk_" + index + ":");
@@ -649,7 +653,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
sb.append("LIVE)");
}
}
- if (i < storages.length - 1) {
+ if (storagesItr.hasNext()) {
sb.append(", ");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org