You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/03/24 19:41:10 UTC
[22/50] [abbrv] hadoop git commit: HDFS-7339. Allocating and
persisting block groups in NameNode. Contributed by Zhe Zhang
HDFS-7339. Allocating and persisting block groups in NameNode. Contributed by Zhe Zhang
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a88a8b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a88a8b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a88a8b5
Branch: refs/heads/HDFS-7285
Commit: 6a88a8b57f461c4fc5d357c6702e8018bb9c8e9e
Parents: 5b810d0
Author: Zhe Zhang <zh...@apache.org>
Authored: Fri Jan 30 16:16:26 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Mar 24 11:16:31 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 4 +
.../server/blockmanagement/BlockIdManager.java | 8 +-
.../SequentialBlockGroupIdGenerator.java | 82 +++++++++++++++++++
.../SequentialBlockIdGenerator.java | 6 +-
.../hdfs/server/namenode/FSDirectory.java | 8 +-
.../hdfs/server/namenode/FSNamesystem.java | 34 +++++---
.../hadoop/hdfs/server/namenode/INodeFile.java | 11 +++
.../hdfs/server/namenode/TestAddBlockgroup.java | 84 ++++++++++++++++++++
9 files changed, 223 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 9ecf242..2b62744 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -219,6 +219,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT = 3;
public static final String DFS_NAMENODE_REPLICATION_MIN_KEY = "dfs.namenode.replication.min";
public static final int DFS_NAMENODE_REPLICATION_MIN_DEFAULT = 1;
+ public static final String DFS_NAMENODE_STRIPE_MIN_KEY = "dfs.namenode.stripe.min";
+ public static final int DFS_NAMENODE_STRIPE_MIN_DEFAULT = 1;
public static final String DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY = "dfs.namenode.replication.pending.timeout-sec";
public static final int DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_DEFAULT = -1;
public static final String DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY = "dfs.namenode.replication.max-streams";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 54c650b..de60b6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -181,4 +181,8 @@ public class HdfsConstants {
public static final byte WARM_STORAGE_POLICY_ID = 5;
public static final byte EC_STORAGE_POLICY_ID = 4;
public static final byte COLD_STORAGE_POLICY_ID = 2;
+
+ public static final byte NUM_DATA_BLOCKS = 3;
+ public static final byte NUM_PARITY_BLOCKS = 2;
+ public static final byte MAX_BLOCKS_IN_GROUP = 16;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
index 1c69203..c8b9d20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
@@ -53,10 +53,12 @@ public class BlockIdManager {
* The global block ID space for this file system.
*/
private final SequentialBlockIdGenerator blockIdGenerator;
+ private final SequentialBlockGroupIdGenerator blockGroupIdGenerator;
public BlockIdManager(BlockManager blockManager) {
this.generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
this.blockIdGenerator = new SequentialBlockIdGenerator(blockManager);
+ this.blockGroupIdGenerator = new SequentialBlockGroupIdGenerator(blockManager);
}
/**
@@ -190,6 +192,10 @@ public class BlockIdManager {
return blockIdGenerator.nextValue();
}
+ public long nextBlockGroupId() {
+ return blockGroupIdGenerator.nextValue();
+ }
+
public boolean isGenStampInFuture(Block block) {
if (isLegacyBlock(block)) {
return block.getGenerationStamp() > getGenerationStampV1();
@@ -205,4 +211,4 @@ public class BlockIdManager {
.LAST_RESERVED_BLOCK_ID);
generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
new file mode 100644
index 0000000..e9e22ee
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.util.SequentialNumber;
+
+/**
+ * Generate the next valid block group ID by incrementing the maximum block
+ * group ID allocated so far, with the first 2^10 block group IDs reserved.
+ * HDFS-EC introduces a hierarchical protocol to name blocks and groups:
+ * Contiguous: {reserved block IDs | flag | block ID}
+ * Striped: {reserved block IDs | flag | block group ID | index in group}
+ *
+ * Following n bits of reserved block IDs, The (n+1)th bit in an ID
+ * distinguishes contiguous (0) and striped (1) blocks. For a striped block,
+ * bits (n+2) to (64-m) represent the ID of its block group, while the last m
+ * bits represent its index of the group. The value m is determined by the
+ * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP).
+ */
+@InterfaceAudience.Private
+public class SequentialBlockGroupIdGenerator extends SequentialNumber {
+
+ private final BlockManager blockManager;
+
+ SequentialBlockGroupIdGenerator(BlockManager blockManagerRef) {
+ super(Long.MIN_VALUE);
+ this.blockManager = blockManagerRef;
+ }
+
+ @Override // NumberGenerator
+ public long nextValue() {
+ // Skip to next legitimate block group ID based on the naming protocol
+ while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP > 0) {
+ super.nextValue();
+ }
+ // Make sure there's no conflict with existing random block IDs
+ while (hasValidBlockInRange(super.getCurrentValue())) {
+ super.skipTo(super.getCurrentValue() +
+ HdfsConstants.MAX_BLOCKS_IN_GROUP);
+ }
+ if (super.getCurrentValue() >= 0) {
+ BlockManager.LOG.warn("All negative block group IDs are used, " +
+ "growing into positive IDs, " +
+ "which might conflict with non-erasure coded blocks.");
+ }
+ return super.getCurrentValue();
+ }
+
+ /**
+ *
+ * @param id The starting ID of the range
+ * @return true if any ID in the range
+ * {id, id+HdfsConstants.MAX_BLOCKS_IN_GROUP} is pointed-to by a file
+ */
+ private boolean hasValidBlockInRange(long id) {
+ for (int i = 0; i < HdfsConstants.MAX_BLOCKS_IN_GROUP; i++) {
+ Block b = new Block(id + i);
+ if (blockManager.getBlockCollection(b) != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
index eef8857..c97de4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.util.SequentialNumber;
/**
@@ -54,6 +53,11 @@ public class SequentialBlockIdGenerator extends SequentialNumber {
while(isValidBlock(b)) {
b.setBlockId(super.nextValue());
}
+ if (b.getBlockId() < 0) {
+ BlockManager.LOG.warn("All positive block IDs are used, " +
+ "wrapping to negative IDs, " +
+ "which might conflict with erasure coded block groups.");
+ }
return b.getBlockId();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 2f73627..34dba0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -467,10 +467,14 @@ public class FSDirectory implements Closeable {
* Add a block to the file. Returns a reference to the added block.
*/
BlockInfoContiguous addBlock(String path, INodesInPath inodesInPath,
- Block block, DatanodeStorageInfo[] targets) throws IOException {
+ Block block, DatanodeStorageInfo[] targets,
+ boolean isStriped) throws IOException {
writeLock();
try {
final INodeFile fileINode = inodesInPath.getLastINode().asFile();
+ short numLocations = isStriped ?
+ HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS :
+ fileINode.getFileReplication();
Preconditions.checkState(fileINode.isUnderConstruction());
// check quota limits and updated space consumed
@@ -481,7 +485,7 @@ public class FSDirectory implements Closeable {
BlockInfoContiguousUnderConstruction blockInfo =
new BlockInfoContiguousUnderConstruction(
block,
- fileINode.getFileReplication(),
+ numLocations,
BlockUCState.UNDER_CONSTRUCTION,
targets);
getBlockManager().addBlockCollection(blockInfo, fileINode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 34b5e95..38c458c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2046,7 +2046,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
BlockInfoContiguous oldBlock = file.getLastBlock();
boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock);
if(newBlock == null) {
- newBlock = (shouldCopyOnTruncate) ? createNewBlock() :
+ newBlock = (shouldCopyOnTruncate) ? createNewBlock(file.isStriped()) :
new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(),
nextGenerationStamp(blockIdManager.isLegacyBlock(oldBlock)));
}
@@ -2991,8 +2991,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
ExtendedBlock previous, Set<Node> excludedNodes,
List<String> favoredNodes) throws IOException {
final long blockSize;
- final int replication;
+ final short numTargets;
final byte storagePolicyID;
+ final boolean isStriped;
Node clientNode = null;
String clientMachine = null;
@@ -3030,7 +3031,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.getClientMachine();
clientNode = blockManager.getDatanodeManager().getDatanodeByHost(
clientMachine);
- replication = pendingFile.getFileReplication();
+ // TODO: make block group size configurable (HDFS-7337)
+ isStriped = pendingFile.isStriped();
+ numTargets = isStriped ?
+ HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS :
+ pendingFile.getFileReplication();
storagePolicyID = pendingFile.getStoragePolicyID();
} finally {
readUnlock();
@@ -3042,7 +3047,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// choose targets for the new block to be allocated.
final DatanodeStorageInfo targets[] = getBlockManager().chooseTarget4NewBlock(
- src, replication, clientNode, excludedNodes, blockSize, favoredNodes,
+ src, numTargets, clientNode, excludedNodes, blockSize, favoredNodes,
storagePolicyID);
// Part II.
@@ -3081,9 +3086,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
ExtendedBlock.getLocalBlock(previous));
// allocate new block, record block locations in INode.
- newBlock = createNewBlock();
+ newBlock = createNewBlock(isStriped);
INodesInPath inodesInPath = INodesInPath.fromINode(pendingFile);
- saveAllocatedBlock(src, inodesInPath, newBlock, targets);
+ saveAllocatedBlock(src, inodesInPath, newBlock, targets, isStriped);
persistNewBlock(src, pendingFile);
offset = pendingFile.computeFileSize();
@@ -3504,13 +3509,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* The last INode is the INode for {@code src} file.
* @param newBlock newly allocated block to be save
* @param targets target datanodes where replicas of the new block is placed
+ * @param isStriped is the file under striping or contigunous layout?
* @throws QuotaExceededException If addition of block exceeds space quota
*/
BlockInfoContiguous saveAllocatedBlock(String src, INodesInPath inodesInPath,
- Block newBlock, DatanodeStorageInfo[] targets)
+ Block newBlock, DatanodeStorageInfo[] targets, boolean isStriped)
throws IOException {
assert hasWriteLock();
- BlockInfoContiguous b = dir.addBlock(src, inodesInPath, newBlock, targets);
+ BlockInfoContiguous b = dir.addBlock(src, inodesInPath, newBlock, targets,
+ isStriped);
NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
DatanodeStorageInfo.incrementBlocksScheduled(targets);
return b;
@@ -3518,10 +3525,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Create new block with a unique block id and a new generation stamp.
+ * @param isStriped is the file under striping or contiguous layout?
*/
- Block createNewBlock() throws IOException {
+ Block createNewBlock(boolean isStriped) throws IOException {
assert hasWriteLock();
- Block b = new Block(nextBlockId(), 0, 0);
+ Block b = new Block(nextBlockId(isStriped), 0, 0);
// Increment the generation stamp for every new block.
b.setGenerationStamp(nextGenerationStamp(false));
return b;
@@ -6125,11 +6133,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Increments, logs and then returns the block ID
+ * @param isStriped is the file under striping or contiguous layout?
*/
- private long nextBlockId() throws IOException {
+ private long nextBlockId(boolean isStriped) throws IOException {
assert hasWriteLock();
checkNameNodeSafeMode("Cannot get next block ID");
- final long blockId = blockIdManager.nextBlockId();
+ final long blockId = isStriped ?
+ blockIdManager.nextBlockGroupId() : blockIdManager.nextBlockId();
getEditLog().logAllocateBlockId(blockId);
// NB: callers sync the log
return blockId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index ae554fe..3772690 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -34,12 +34,14 @@ import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
@@ -904,4 +906,13 @@ public class INodeFile extends INodeWithAdditionalFields
return snapshotBlocks != null &&
Arrays.asList(snapshotBlocks).contains(block);
}
+
+ @VisibleForTesting
+ /**
+ * @return true if the file is in the striping layout.
+ */
+ // TODO: move erasure coding policy to file XAttr (HDFS-7337)
+ public boolean isStriped() {
+ return getStoragePolicyID() == HdfsConstants.EC_STORAGE_POLICY_ID;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a88a8b5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java
new file mode 100644
index 0000000..95133ce
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestAddBlockgroup {
+
+ public static final Log LOG = LogFactory.getLog(TestAddBlockgroup.class);
+
+ private final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS +
+ HdfsConstants.NUM_PARITY_BLOCKS;
+ private final short NUM_DATANODES = GROUP_SIZE;
+
+ private static final int BLOCKSIZE = 1024;
+ private static final short REPLICATION = 3;
+
+ private MiniDFSCluster cluster;
+ private Configuration conf;
+
+ @Before
+ public void setup() throws IOException {
+ conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES)
+ .build();
+ cluster.waitActive();
+ cluster.getFileSystem().setStoragePolicy(new Path("/"),
+ HdfsConstants.EC_STORAGE_POLICY_NAME);
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAddBlockGroup() throws Exception {
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
+
+ final Path file1 = new Path("/file1");
+ DFSTestUtil.createFile(fs, file1, BLOCKSIZE * 2, REPLICATION, 0L);
+ INodeFile file1Node = fsdir.getINode4Write(file1.toString()).asFile();
+ BlockInfo[] file1Blocks = file1Node.getBlocks();
+ assertEquals(2, file1Blocks.length);
+ assertEquals(GROUP_SIZE, file1Blocks[0].numNodes());
+ assertEquals(HdfsConstants.MAX_BLOCKS_IN_GROUP,
+ file1Blocks[1].getBlockId() - file1Blocks[0].getBlockId());
+ }
+}