You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/01/26 21:12:37 UTC
[43/57] [abbrv] hadoop git commit: HDFS-11124. Report blockIds of
internal blocks for EC files in Fsck. Contributed by Takanobu Asanuma.
HDFS-11124. Report blockIds of internal blocks for EC files in Fsck. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b782bf21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b782bf21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b782bf21
Branch: refs/heads/YARN-5734
Commit: b782bf2156dd9d43610c0bc47d458b2db297589f
Parents: 18e1d68
Author: Jing Zhao <ji...@apache.org>
Authored: Wed Jan 25 11:16:17 2017 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Wed Jan 25 11:16:17 2017 -0800
----------------------------------------------------------------------
.../blockmanagement/BlockInfoStriped.java | 14 +++++--
.../server/blockmanagement/BlockManager.java | 6 +--
.../hdfs/server/namenode/NamenodeFsck.java | 16 +++++++
.../hadoop/hdfs/server/namenode/TestFsck.java | 44 +++++++++++++++-----
4 files changed, 63 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b782bf21/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
index 86e42a6..790cd77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
@@ -244,14 +244,22 @@ public class BlockInfoStriped extends BlockInfo {
return true;
}
- static class StorageAndBlockIndex {
- final DatanodeStorageInfo storage;
- final byte blockIndex;
+ public static class StorageAndBlockIndex {
+ private final DatanodeStorageInfo storage;
+ private final byte blockIndex;
StorageAndBlockIndex(DatanodeStorageInfo storage, byte blockIndex) {
this.storage = storage;
this.blockIndex = blockIndex;
}
+
+ public DatanodeStorageInfo getStorage() {
+ return storage;
+ }
+
+ public byte getBlockIndex() {
+ return blockIndex;
+ }
}
public Iterable<StorageAndBlockIndex> getStorageAndIndexInfos() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b782bf21/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index a2fd843..893b12d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3903,10 +3903,10 @@ public class BlockManager implements BlockStatsMXBean {
BitSet bitSet = new BitSet(block.getTotalBlockNum());
for (StorageAndBlockIndex si : block.getStorageAndIndexInfos()) {
StoredReplicaState state = checkReplicaOnStorage(counters, block,
- si.storage, nodesCorrupt, inStartupSafeMode);
+ si.getStorage(), nodesCorrupt, inStartupSafeMode);
if (state == StoredReplicaState.LIVE) {
- if (!bitSet.get(si.blockIndex)) {
- bitSet.set(si.blockIndex);
+ if (!bitSet.get(si.getBlockIndex())) {
+ bitSet.set(si.getBlockIndex());
} else {
counters.subtract(StoredReplicaState.LIVE, 1);
counters.add(StoredReplicaState.REDUNDANT, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b782bf21/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 1ae75f0..859f541 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -28,6 +28,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -67,6 +68,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped.StorageAndBlockIndex;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicies;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
@@ -591,9 +593,23 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
blockManager.getStorages(storedBlock) :
storedBlock.getUnderConstructionFeature().getExpectedStorageLocations();
StringBuilder sb = new StringBuilder(" [");
+ final boolean isStriped = storedBlock.isStriped();
+ Map<DatanodeStorageInfo, Long> storage2Id = new HashMap<>();
+ if (isStriped && isComplete) {
+ long blockId = storedBlock.getBlockId();
+ Iterable<StorageAndBlockIndex> sis =
+ ((BlockInfoStriped)storedBlock).getStorageAndIndexInfos();
+ for (StorageAndBlockIndex si: sis){
+ storage2Id.put(si.getStorage(), blockId + si.getBlockIndex());
+ }
+ }
for (int i = 0; i < storages.length; i++) {
DatanodeStorageInfo storage = storages[i];
+ if (isStriped && isComplete) {
+ long index = storage2Id.get(storage);
+ sb.append("blk_" + index + ":");
+ }
DatanodeDescriptor dnDesc = storage.getDatanodeDescriptor();
if (showRacks) {
sb.append(NodeBase.getPath(dnDesc));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b782bf21/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 61ecae3..c02e880 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -677,16 +677,16 @@ public class TestFsck {
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
ErasureCodingPolicy ecPolicy =
ErasureCodingPolicyManager.getSystemDefaultPolicy();
- int numAllUnits = ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits();
+ final int dataBlocks = ecPolicy.getNumDataUnits();
+ final int cellSize = ecPolicy.getCellSize();
+ final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits();
+ int blockSize = 2 * cellSize;
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
numAllUnits + 1).build();
- FileSystem fs = null;
String topDir = "/myDir";
- byte[] randomBytes = new byte[3000000];
- int seed = 42;
- new Random(seed).nextBytes(randomBytes);
cluster.waitActive();
- fs = cluster.getFileSystem();
+ DistributedFileSystem fs = cluster.getFileSystem();
util.createFiles(fs, topDir);
// set topDir to EC when it has replicated files
cluster.getFileSystem().getClient().setErasureCodingPolicy(
@@ -697,11 +697,12 @@ public class TestFsck {
// Open a EC file for writing and do not close for now
Path openFile = new Path(topDir + "/openECFile");
FSDataOutputStream out = fs.create(openFile);
- int writeCount = 0;
- while (writeCount != 300) {
- out.write(randomBytes);
- writeCount++;
- }
+ int blockGroupSize = dataBlocks * blockSize;
+ // data size is more than 1 block group and less than 2 block groups
+ byte[] randomBytes = new byte[2 * blockGroupSize - cellSize];
+ int seed = 42;
+ new Random(seed).nextBytes(randomBytes);
+ out.write(randomBytes);
// make sure the fsck can correctly handle mixed ec/replicated files
runFsck(conf, 0, true, topDir, "-files", "-blocks", "-openforwrite");
@@ -723,6 +724,27 @@ public class TestFsck {
assertTrue(outStr.contains("Expected_repl=" + numAllUnits));
assertTrue(outStr.contains("Under Construction Block:"));
+ // check reported blockIDs of internal blocks
+ LocatedStripedBlock lsb = (LocatedStripedBlock)fs.getClient()
+ .getLocatedBlocks(openFile.toString(), 0, cellSize * dataBlocks).get(0);
+ long groupId = lsb.getBlock().getBlockId();
+ byte[] indices = lsb.getBlockIndices();
+ DatanodeInfo[] locs = lsb.getLocations();
+ long blockId;
+ for (int i = 0; i < indices.length; i++) {
+ blockId = groupId + indices[i];
+ String str = "blk_" + blockId + ":" + locs[i];
+ assertTrue(outStr.contains(str));
+ }
+
+ // check the output of under-constructed blocks doesn't include the blockIDs
+ String regex = ".*Expected_repl=" + numAllUnits + "(.*)\nStatus:.*";
+ Pattern p = Pattern.compile(regex, Pattern.DOTALL);
+ Matcher m = p.matcher(outStr);
+ assertTrue(m.find());
+ String ucBlockOutput = m.group(1);
+ assertFalse(ucBlockOutput.contains("blk_"));
+
// Close the file
out.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org