You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wh...@apache.org on 2016/01/05 20:52:05 UTC
[05/50] [abbrv] hadoop git commit: HDFS-8248. Store INodeId instead
of the INodeFile object in BlockInfoContiguous.
HDFS-8248. Store INodeId instead of the INodeFile object in BlockInfoContiguous.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72e1828b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72e1828b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72e1828b
Branch: refs/heads/feature-HDFS-8286
Commit: 72e1828bf5afb1ff43a0202783075dcafa266b28
Parents: cf23b98
Author: Haohui Mai <wh...@apache.org>
Authored: Fri Apr 24 15:09:04 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Fri Jun 12 13:56:56 2015 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockCollection.java | 5 ++
.../blockmanagement/BlockInfoContiguous.java | 24 ++++---
.../server/blockmanagement/BlockManager.java | 75 +++++++++++---------
.../hdfs/server/blockmanagement/BlocksMap.java | 10 +--
.../blockmanagement/DecommissionManager.java | 11 +--
.../SequentialBlockIdGenerator.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 27 ++++---
.../hadoop/hdfs/server/namenode/INodeFile.java | 4 +-
.../hadoop/hdfs/server/namenode/INodeId.java | 1 +
.../hdfs/server/namenode/NamenodeFsck.java | 4 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 3 +
.../server/blockmanagement/TestBlockInfo.java | 8 +--
.../blockmanagement/TestBlockManager.java | 9 +++
.../blockmanagement/TestReplicationPolicy.java | 10 ++-
.../TestCommitBlockSynchronization.java | 6 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 33 +++++----
.../snapshot/TestSnapshotBlocksMap.java | 12 ++--
.../namenode/snapshot/TestSnapshotDeletion.java | 9 +--
18 files changed, 154 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index 0ee0439..2d3a0f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -80,4 +80,9 @@ public interface BlockCollection {
* @return whether the block collection is under construction.
*/
public boolean isUnderConstruction();
+
+ /**
+ * @return the INodeId for the block collection
+ */
+ long getId();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index 7a78708..7e0699e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.util.LightWeightGSet;
+import static org.apache.hadoop.hdfs.server.namenode.INodeId.INVALID_INODE_ID;
+
/**
* BlockInfo class maintains for a given block
* the {@link BlockCollection} it is part of and datanodes where the replicas of
@@ -34,7 +36,7 @@ public class BlockInfoContiguous extends Block
implements LightWeightGSet.LinkedElement {
public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
- private BlockCollection bc;
+ private long bcId;
/** For implementing {@link LightWeightGSet.LinkedElement} interface */
private LightWeightGSet.LinkedElement nextLinkedElement;
@@ -61,14 +63,14 @@ public class BlockInfoContiguous extends Block
*/
public BlockInfoContiguous(short replication) {
this.triplets = new Object[3*replication];
- this.bc = null;
+ this.bcId = INVALID_INODE_ID;
this.replication = replication;
}
public BlockInfoContiguous(Block blk, short replication) {
super(blk);
this.triplets = new Object[3*replication];
- this.bc = null;
+ this.bcId = INVALID_INODE_ID;
this.replication = replication;
}
@@ -79,7 +81,7 @@ public class BlockInfoContiguous extends Block
*/
protected BlockInfoContiguous(BlockInfoContiguous from) {
this(from, from.getReplication());
- this.bc = from.bc;
+ this.bcId = from.bcId;
}
public void setReplication(short replication) {
@@ -90,16 +92,16 @@ public class BlockInfoContiguous extends Block
return replication;
}
- public BlockCollection getBlockCollection() {
- return bc;
+ public long getBlockCollectionId() {
+ return bcId;
}
- public void setBlockCollection(BlockCollection bc) {
- this.bc = bc;
+ public void setBlockCollectionId(long bcId) {
+ this.bcId = bcId;
}
public boolean isDeleted() {
- return (bc == null);
+ return bcId == INVALID_INODE_ID;
}
public DatanodeDescriptor getDatanode(int index) {
@@ -374,7 +376,7 @@ public class BlockInfoContiguous extends Block
BlockInfoContiguousUnderConstruction ucBlock =
new BlockInfoContiguousUnderConstruction(this,
getReplication(), s, targets);
- ucBlock.setBlockCollection(getBlockCollection());
+ ucBlock.setBlockCollectionId(getBlockCollectionId());
return ucBlock;
}
// the block is already under construction
@@ -382,7 +384,7 @@ public class BlockInfoContiguous extends Block
(BlockInfoContiguousUnderConstruction)this;
ucBlock.setBlockUCState(s);
ucBlock.setExpectedLocations(targets);
- ucBlock.setBlockCollection(getBlockCollection());
+ ucBlock.setBlockCollectionId(getBlockCollectionId());
return ucBlock;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 2e9b5b66..6139e37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBloc
import org.apache.hadoop.hdfs.server.blockmanagement.PendingDataNodeMessages.ReportedBlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
@@ -542,7 +543,8 @@ public class BlockManager {
numReplicas.decommissionedAndDecommissioning();
if (block instanceof BlockInfoContiguous) {
- BlockCollection bc = ((BlockInfoContiguous) block).getBlockCollection();
+ long bcId = ((BlockInfoContiguous) block).getBlockCollectionId();
+ BlockCollection bc = namesystem.getBlockCollection(bcId);
String fileName = (bc == null) ? "[orphaned]" : bc.getName();
out.print(fileName + ": ");
}
@@ -1275,7 +1277,8 @@ public class BlockManager {
for (int priority = 0; priority < blocksToReplicate.size(); priority++) {
for (Block block : blocksToReplicate.get(priority)) {
// block should belong to a file
- bc = blocksMap.getBlockCollection(block);
+ long bcId = getBlockCollectionId(block);
+ bc = namesystem.getBlockCollection(bcId);
BlockInfoContiguous bi = getStoredBlock(block);
// abandoned block or block reopened for append
if (bc == null || bi == null
@@ -1364,7 +1367,8 @@ public class BlockManager {
int priority = rw.priority;
// Recheck since global lock was released
// block should belong to a file
- bc = blocksMap.getBlockCollection(block);
+ long bcId = getBlockCollectionId(block);
+ bc = namesystem.getBlockCollection(bcId);
// abandoned block or block reopened for append
if(bc == null || (bc.isUnderConstruction() && block.equals(bc.getLastBlock()))) {
neededReplications.remove(block, priority); // remove from neededReplications
@@ -1806,18 +1810,18 @@ public class BlockManager {
if (metrics != null) {
metrics.addBlockReport((int) (endTime - startTime));
}
- blockLog.info("BLOCK* processReport: from storage {} node {}, " +
- "blocks: {}, hasStaleStorage: {}, processing time: {} msecs", storage
- .getStorageID(), nodeID, newReport.getNumberOfBlocks(),
+ blockLog.info(
+ "BLOCK* processReport: from storage {} node {}, " + "blocks: {}, hasStaleStorage: {}, processing time: {} msecs",
+ storage.getStorageID(), nodeID, newReport.getNumberOfBlocks(),
node.hasStaleStorages(), (endTime - startTime));
return !node.hasStaleStorages();
}
private void removeZombieReplicas(BlockReportContext context,
DatanodeStorageInfo zombie) {
- LOG.warn("processReport 0x{}: removing zombie storage {}, which no " +
- "longer exists on the DataNode.",
- Long.toHexString(context.getReportId()), zombie.getStorageID());
+ LOG.warn(
+ "processReport 0x{}: removing zombie storage {}, which no " + "longer exists on the DataNode.",
+ Long.toHexString(context.getReportId()), zombie.getStorageID());
assert(namesystem.hasWriteLock());
Iterator<BlockInfoContiguous> iter = zombie.getBlockIterator();
int prevBlocks = zombie.numBlocks();
@@ -1833,10 +1837,10 @@ public class BlockManager {
invalidateBlocks.remove(zombie.getDatanodeDescriptor(), block);
}
assert(zombie.numBlocks() == 0);
- LOG.warn("processReport 0x{}: removed {} replicas from storage {}, " +
- "which no longer exists on the DataNode.",
- Long.toHexString(context.getReportId()), prevBlocks,
- zombie.getStorageID());
+ LOG.warn(
+ "processReport 0x{}: removed {} replicas from storage {}, " + "which no longer exists on the DataNode.",
+ Long.toHexString(context.getReportId()), prevBlocks,
+ zombie.getStorageID());
}
/**
@@ -2441,7 +2445,9 @@ public class BlockManager {
int numCurrentReplica = countLiveNodes(storedBlock);
if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED
&& numCurrentReplica >= minReplication) {
- completeBlock(storedBlock.getBlockCollection(), storedBlock, false);
+ long bcId = storedBlock.getBlockCollectionId();
+ BlockCollection bc = namesystem.getBlockCollection(bcId);
+ completeBlock(bc, storedBlock, false);
} else if (storedBlock.isComplete() && result == AddBlockResult.ADDED) {
// check whether safe replication is reached for the block
// only complete blocks are counted towards that.
@@ -2479,7 +2485,8 @@ public class BlockManager {
// it will happen in next block report otherwise.
return block;
}
- BlockCollection bc = storedBlock.getBlockCollection();
+ long bcId = storedBlock.getBlockCollectionId();
+ BlockCollection bc = namesystem.getBlockCollection(bcId);
assert bc != null : "Block must belong to a file";
// add block to the datanode
@@ -2813,8 +2820,8 @@ public class BlockManager {
+ " for " + src);
processOverReplicatedBlock(b, newRepl, null, null);
} else { // replication factor is increased
- LOG.info("Increasing replication from " + oldRepl + " to " + newRepl
- + " for " + src);
+ LOG.info(
+ "Increasing replication from " + oldRepl + " to " + newRepl + " for " + src);
}
}
@@ -2881,7 +2888,8 @@ public class BlockManager {
BlockPlacementPolicy replicator) {
assert namesystem.hasWriteLock();
// first form a rack to datanodes map and
- BlockCollection bc = getBlockCollection(b);
+ long bcId = getBlockCollectionId(b);
+ BlockCollection bc = namesystem.getBlockCollection(bcId);
final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(bc.getStoragePolicyID());
final List<StorageType> excessTypes = storagePolicy.chooseExcess(
replication, DatanodeStorageInfo.toStorageTypes(nonExcess));
@@ -3004,8 +3012,8 @@ public class BlockManager {
// necessary. In that case, put block on a possibly-will-
// be-replicated list.
//
- BlockCollection bc = blocksMap.getBlockCollection(block);
- if (bc != null) {
+ long bcId = getBlockCollectionId(block);
+ if (bcId != INodeId.INVALID_INODE_ID) {
namesystem.decrementSafeBlockCount(block);
updateNeededReplications(block, -1, 0);
}
@@ -3097,8 +3105,8 @@ public class BlockManager {
Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>();
final DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
- processReportedBlock(storageInfo, block, reportedState,
- toAdd, toInvalidate, toCorrupt, toUC);
+ processReportedBlock(storageInfo, block, reportedState, toAdd, toInvalidate,
+ toCorrupt, toUC);
// the block is only in one of the to-do lists
// if it is in none then data-node already has it
assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1
@@ -3183,7 +3191,7 @@ public class BlockManager {
break;
}
blockLog.debug("BLOCK* block {}: {} is received from {}",
- rdbi.getStatus(), rdbi.getBlock(), nodeID);
+ rdbi.getStatus(), rdbi.getBlock(), nodeID);
}
blockLog.debug("*BLOCK* NameNode.processIncrementalBlockReport: from "
+ "{} receiving: {}, received: {}, deleted: {}", nodeID, receiving,
@@ -3274,8 +3282,9 @@ public class BlockManager {
numOverReplicated++;
}
}
- LOG.info("Invalidated " + numOverReplicated + " over-replicated blocks on " +
- srcNode + " during recommissioning");
+ LOG.info(
+ "Invalidated " + numOverReplicated + " over-replicated blocks on " +
+ srcNode + " during recommissioning");
}
/**
@@ -3302,11 +3311,11 @@ public class BlockManager {
}
LOG.warn("Node {} is dead " +
- "while decommission is in progress. Cannot be safely " +
- "decommissioned since there is risk of reduced " +
- "data durability or data loss. Either restart the failed node or" +
- " force decommissioning by removing, calling refreshNodes, " +
- "then re-adding to the excludes files.", node);
+ "while decommission is in progress. Cannot be safely " +
+ "decommissioned since there is risk of reduced " +
+ "data durability or data loss. Either restart the failed node or" +
+ " force decommissioning by removing, calling refreshNodes, " +
+ "then re-adding to the excludes files.", node);
return false;
}
@@ -3417,6 +3426,7 @@ public class BlockManager {
* otherwise, return the replication factor of the block.
*/
private int getReplication(Block block) {
+ assert namesystem.hasReadLock();
BlockInfoContiguous bi = blocksMap.getStoredBlock(block);
return bi == null ? 0 : bi.getReplication();
}
@@ -3517,8 +3527,9 @@ public class BlockManager {
return blocksMap.addBlockCollection(block, bc);
}
- public BlockCollection getBlockCollection(Block b) {
- return blocksMap.getBlockCollection(b);
+ public long getBlockCollectionId(Block b) {
+ BlockInfoContiguous bi = getStoredBlock(b);
+ return bi == null ? INodeId.INVALID_INODE_ID : bi.getBlockCollectionId();
}
/** @return an iterator of the datanodes. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index 5e7d34f..9a1dc29 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -21,6 +21,7 @@ import java.util.Iterator;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
@@ -96,11 +97,6 @@ class BlocksMap {
}
}
- BlockCollection getBlockCollection(Block b) {
- BlockInfoContiguous info = blocks.get(b);
- return (info != null) ? info.getBlockCollection() : null;
- }
-
/**
* Add block b belonging to the specified block collection to the map.
*/
@@ -110,7 +106,7 @@ class BlocksMap {
info = b;
blocks.put(info);
}
- info.setBlockCollection(bc);
+ info.setBlockCollectionId(bc.getId());
return info;
}
@@ -124,7 +120,7 @@ class BlocksMap {
if (blockInfo == null)
return;
- blockInfo.setBlockCollection(null);
+ blockInfo.setBlockCollectionId(INodeId.INVALID_INODE_ID);
for(int idx = blockInfo.numNodes()-1; idx >= 0; idx--) {
DatanodeDescriptor dn = blockInfo.getDatanode(idx);
dn.removeBlock(blockInfo); // remove from the list and wipe the location
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 48fb39c..172c1e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.util.CyclicIteration;
import org.apache.hadoop.util.ChunkedArrayList;
@@ -553,12 +554,14 @@ public class DecommissionManager {
it.remove();
continue;
}
- BlockCollection bc = blockManager.blocksMap.getBlockCollection(block);
- if (bc == null) {
+
+ long bcId = block.getBlockCollectionId();
+ if (bcId == INodeId.INVALID_INODE_ID) {
// Orphan block, will be invalidated eventually. Skip.
continue;
}
+ BlockCollection bc = namesystem.getBlockCollection(bcId);
final NumberReplicas num = blockManager.countNodes(block);
final int liveReplicas = num.liveReplicas();
final int curReplicas = liveReplicas;
@@ -608,8 +611,8 @@ public class DecommissionManager {
}
datanode.decommissioningStatus.set(underReplicatedBlocks,
- decommissionOnlyReplicas,
- underReplicatedInOpenFiles);
+ decommissionOnlyReplicas,
+ underReplicatedInOpenFiles);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
index eef8857..9e7f1f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.util.SequentialNumber;
/**
@@ -61,6 +62,6 @@ public class SequentialBlockIdGenerator extends SequentialNumber {
* Returns whether the given block is one pointed-to by a file.
*/
private boolean isValidBlock(Block b) {
- return (blockManager.getBlockCollection(b) != null);
+ return blockManager.getBlockCollectionId(b) != INodeId.INVALID_INODE_ID;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index dbf2d63..ed28547 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3567,7 +3567,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
@Override
public boolean isInSnapshot(BlockInfoContiguousUnderConstruction blockUC) {
assert hasReadLock();
- final BlockCollection bc = blockUC.getBlockCollection();
+ long bcId = blockUC.getBlockCollectionId();
+ final BlockCollection bc = getBlockCollection(bcId);
if (bc == null || !(bc instanceof INodeFile)
|| !bc.isUnderConstruction()) {
return false;
@@ -3596,6 +3597,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return true;
}
+ @Override
+ public BlockCollection getBlockCollection(long id) {
+ INode inode = getFSDirectory().getInode(id);
+ return inode == null ? null : inode.asFile();
+ }
+
void commitBlockSynchronization(ExtendedBlock oldBlock,
long newgenerationstamp, long newlength,
boolean closeFile, boolean deleteblock, DatanodeID[] newtargets,
@@ -3653,7 +3660,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
+ " is null, likely because the file owning this block was"
+ " deleted and the block removal is delayed");
}
- INodeFile iFile = ((INode)storedBlock.getBlockCollection()).asFile();
+ long bcId = storedBlock.getBlockCollectionId();
+ INodeFile iFile = ((INode)getBlockCollection(bcId)).asFile();
if (isFileDeleted(iFile)) {
throw new FileNotFoundException("File not found: "
+ iFile.getFullPathName() + ", likely due to delayed block"
@@ -4073,9 +4081,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
while (it.hasNext()) {
Block b = it.next();
BlockInfoContiguous blockInfo = blockManager.getStoredBlock(b);
- if (blockInfo.getBlockCollection().getStoragePolicyID()
- == lpPolicy.getId()) {
- filesToDelete.add(blockInfo.getBlockCollection());
+ long bcId = blockInfo.getBlockCollectionId();
+ BlockCollection bc = getBlockCollection(bcId);
+ if (bc.getStoragePolicyID() == lpPolicy.getId()) {
+ filesToDelete.add(bc);
}
}
} finally {
@@ -5571,7 +5580,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
// check file inode
- final INodeFile file = ((INode)storedBlock.getBlockCollection()).asFile();
+ long bcId = storedBlock.getBlockCollectionId();
+ final INodeFile file = ((INode)getBlockCollection(bcId)).asFile();
if (file == null || !file.isUnderConstruction() || isFileDeleted(file)) {
throw new IOException("The file " + storedBlock +
" belonged to does not exist or it is not under construction.");
@@ -5831,8 +5841,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
while (blkIterator.hasNext()) {
- Block blk = blkIterator.next();
- final INode inode = (INode)blockManager.getBlockCollection(blk);
+ BlockInfoContiguous blk = (BlockInfoContiguous) blkIterator.next();
+ final INode inode = (INode)
+ getBlockCollection(blk.getBlockCollectionId());
skip++;
if (inode != null && blockManager.countNodes(blk).liveReplicas() == 0) {
String src = FSDirectory.getFullPathName(inode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 60f3ad6..1fdbcc3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -428,7 +428,7 @@ public class INodeFile extends INodeWithAdditionalFields
void updateBlockCollection() {
if (blocks != null) {
for(BlockInfoContiguous b : blocks) {
- b.setBlockCollection(this);
+ b.setBlockCollectionId(getId());
}
}
}
@@ -526,7 +526,7 @@ public class INodeFile extends INodeWithAdditionalFields
if (blocks != null && reclaimContext.collectedBlocks != null) {
for (BlockInfoContiguous blk : blocks) {
reclaimContext.collectedBlocks.addDeleteBlock(blk);
- blk.setBlockCollection(null);
+ blk.setBlockCollectionId(INodeId.INVALID_INODE_ID);
}
}
setBlocks(null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
index 00b33cd..10139bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
@@ -37,6 +37,7 @@ public class INodeId extends SequentialNumber {
*/
public static final long LAST_RESERVED_ID = 2 << 14 - 1;
public static final long ROOT_INODE_ID = LAST_RESERVED_ID + 1;
+ public static final long INVALID_INODE_ID = -1;
/**
* To check if the request id is the same as saved id. Don't check fileId
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index be88e10..566ddf3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -249,8 +249,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
LOG.warn("Block "+ blockId + " " + NONEXISTENT_STATUS);
return;
}
- BlockCollection bc = bm.getBlockCollection(blockInfo);
- INode iNode = (INode) bc;
+ INode iNode = (INode) namenode.getNamesystem().getBlockCollection(
+ blockInfo.getBlockCollectionId());
NumberReplicas numberReplicas= bm.countNodes(block);
out.println("Block Id: " + blockId);
out.println("Block belongs to: "+iNode.getFullPathName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
index 3442e7b..7f4c3fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.util.RwLock;
@@ -46,4 +47,6 @@ public interface Namesystem extends RwLock, SafeMode {
public void checkOperation(OperationCategory read) throws StandbyException;
public boolean isInSnapshot(BlockInfoContiguousUnderConstruction blockUC);
+
+ BlockCollection getBlockCollection(long id);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index c5662ca..5adf728 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.junit.Assert;
import org.junit.Test;
@@ -50,10 +51,9 @@ public class TestBlockInfo {
@Test
public void testIsDeleted() {
BlockInfoContiguous blockInfo = new BlockInfoContiguous((short) 3);
- BlockCollection bc = Mockito.mock(BlockCollection.class);
- blockInfo.setBlockCollection(bc);
+ blockInfo.setBlockCollectionId(1000);
Assert.assertFalse(blockInfo.isDeleted());
- blockInfo.setBlockCollection(null);
+ blockInfo.setBlockCollectionId(INodeId.INVALID_INODE_ID);
Assert.assertTrue(blockInfo.isDeleted());
}
@@ -74,7 +74,7 @@ public class TestBlockInfo {
BlockInfoContiguous old = new BlockInfoContiguous((short) 3);
try {
BlockInfoContiguous copy = new BlockInfoContiguous(old);
- assertEquals(old.getBlockCollection(), copy.getBlockCollection());
+ assertEquals(old.getBlockCollectionId(), copy.getBlockCollectionId());
assertEquals(old.getCapacity(), copy.getCapacity());
} catch (Exception e) {
Assert.fail("Copy constructor throws exception: " + e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index d9869ec..4229713 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -88,6 +89,7 @@ public class TestBlockManager {
private FSNamesystem fsn;
private BlockManager bm;
+ private long mockINodeId;
@Before
public void setupMockCluster() throws IOException {
@@ -96,6 +98,7 @@ public class TestBlockManager {
"need to set a dummy value here so it assumes a multi-rack cluster");
fsn = Mockito.mock(FSNamesystem.class);
Mockito.doReturn(true).when(fsn).hasWriteLock();
+ Mockito.doReturn(true).when(fsn).hasReadLock();
bm = new BlockManager(fsn, conf);
final String[] racks = {
"/rackA",
@@ -108,6 +111,7 @@ public class TestBlockManager {
nodes = Arrays.asList(DFSTestUtil.toDatanodeDescriptor(storages));
rackA = nodes.subList(0, 3);
rackB = nodes.subList(3, 6);
+ mockINodeId = INodeId.ROOT_INODE_ID + 1;
}
private void addNodes(Iterable<DatanodeDescriptor> nodesToAdd) {
@@ -432,8 +436,13 @@ public class TestBlockManager {
}
private BlockInfoContiguous addBlockOnNodes(long blockId, List<DatanodeDescriptor> nodes) {
+ long inodeId = mockINodeId++;
BlockCollection bc = Mockito.mock(BlockCollection.class);
+ Mockito.doReturn(inodeId).when(bc).getId();
+ Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId);
BlockInfoContiguous blockInfo = blockOnNodes(blockId, nodes);
+ blockInfo.setReplication((short) 3);
+ blockInfo.setBlockCollectionId(inodeId);
bm.blocksMap.addBlockCollection(blockInfo, bc);
return blockInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index d3df32c..56c1311 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1154,6 +1154,7 @@ public class TestReplicationPolicy {
Namesystem mockNS = mock(Namesystem.class);
when(mockNS.isPopulatingReplQueues()).thenReturn(true);
when(mockNS.hasWriteLock()).thenReturn(true);
+ when(mockNS.hasReadLock()).thenReturn(true);
BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
@@ -1179,6 +1180,8 @@ public class TestReplicationPolicy {
// queue.
BlockInfoContiguousUnderConstruction info = new BlockInfoContiguousUnderConstruction(block1, (short) 1);
BlockCollection bc = mock(BlockCollection.class);
+ when(bc.getId()).thenReturn(1000L);
+ when(mockNS.getBlockCollection(1000L)).thenReturn(bc);
bm.addBlockCollection(info, bc);
// Adding this block will increase its current replication, and that will
@@ -1199,6 +1202,8 @@ public class TestReplicationPolicy {
throws IOException {
Namesystem mockNS = mock(Namesystem.class);
when(mockNS.isPopulatingReplQueues()).thenReturn(true);
+ when(mockNS.hasReadLock()).thenReturn(true);
+
BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
@@ -1220,13 +1225,14 @@ public class TestReplicationPolicy {
final BlockInfoContiguous info = new BlockInfoContiguous(block1, (short) 1);
final BlockCollection mbc = mock(BlockCollection.class);
+ when(mbc.getId()).thenReturn(1000L);
when(mbc.getLastBlock()).thenReturn(info);
when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);
when(mbc.isUnderConstruction()).thenReturn(true);
ContentSummary cs = mock(ContentSummary.class);
when(cs.getLength()).thenReturn((long)1);
when(mbc.computeContentSummary(bm.getStoragePolicySuite())).thenReturn(cs);
- info.setBlockCollection(mbc);
+ info.setBlockCollectionId(1000);
bm.addBlockCollection(info, mbc);
DatanodeStorageInfo[] storageAry = {new DatanodeStorageInfo(
@@ -1261,6 +1267,8 @@ public class TestReplicationPolicy {
throws IOException {
Namesystem mockNS = mock(Namesystem.class);
when(mockNS.isPopulatingReplQueues()).thenReturn(true);
+ when(mockNS.hasReadLock()).thenReturn(true);
+
BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index ea560fe..629b93c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -69,7 +69,7 @@ public class TestCommitBlockSynchronization {
FSNamesystem namesystemSpy = spy(namesystem);
BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction(
block, (short) 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
- blockInfo.setBlockCollection(file);
+ blockInfo.setBlockCollectionId(file.getId());
blockInfo.setGenerationStamp(genStamp);
blockInfo.initializeBlockRecovery(genStamp);
doReturn(blockInfo).when(file).removeLastBlock(any(Block.class));
@@ -107,7 +107,7 @@ public class TestCommitBlockSynchronization {
// Simulate 'completing' the block.
BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
- completedBlockInfo.setBlockCollection(file);
+ completedBlockInfo.setBlockCollectionId(file.getId());
completedBlockInfo.setGenerationStamp(genStamp);
doReturn(completedBlockInfo).when(namesystemSpy)
.getStoredBlock(any(Block.class));
@@ -179,7 +179,7 @@ public class TestCommitBlockSynchronization {
lastBlock, genStamp, length, true, false, newTargets, null);
BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
- completedBlockInfo.setBlockCollection(file);
+ completedBlockInfo.setBlockCollectionId(file.getId());
completedBlockInfo.setGenerationStamp(genStamp);
doReturn(completedBlockInfo).when(namesystemSpy)
.getStoredBlock(any(Block.class));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index c2899fd..26f91ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -826,11 +826,12 @@ public class TestFsck {
// decommission datanode
ExtendedBlock eb = DFSTestUtil.getFirstBlock(dfs, path);
- DatanodeDescriptor dn =
- cluster.getNameNode().getNamesystem().getBlockManager()
- .getBlockCollection(eb.getLocalBlock()).getBlocks()[0].getDatanode(0);
- cluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager()
- .getDecomManager().startDecommission(dn);
+ FSNamesystem fsn = cluster.getNameNode().getNamesystem();
+ BlockManager bm = fsn.getBlockManager();
+ long bcId = bm.getBlockCollectionId(eb.getLocalBlock());
+ DatanodeDescriptor dn = fsn.getBlockCollection(bcId).getBlocks()[0]
+ .getDatanode(0);
+ bm.getDatanodeManager().getDecomManager().startDecommission(dn);
String dnName = dn.getXferAddr();
// check the replica status while decommissioning
@@ -1385,12 +1386,13 @@ public class TestFsck {
assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
//decommission datanode
+ FSNamesystem fsn = cluster.getNameNode().getNamesystem();
+ BlockManager bm = fsn.getBlockManager();
ExtendedBlock eb = util.getFirstBlock(dfs, path);
- DatanodeDescriptor dn = cluster.getNameNode().getNamesystem()
- .getBlockManager().getBlockCollection(eb.getLocalBlock())
- .getBlocks()[0].getDatanode(0);
- cluster.getNameNode().getNamesystem().getBlockManager()
- .getDatanodeManager().getDecomManager().startDecommission(dn);
+ long bcId = bm.getBlockCollectionId(eb.getLocalBlock());
+ DatanodeDescriptor dn = fsn.getBlockCollection(bcId).getBlocks()[0]
+ .getDatanode(0);
+ bm.getDatanodeManager().getDecomManager().startDecommission(dn);
String dnName = dn.getXferAddr();
//wait for decommission start
@@ -1593,12 +1595,13 @@ public class TestFsck {
assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
// decommission datanode
+ FSNamesystem fsn = cluster.getNameNode().getNamesystem();
+ BlockManager bm = fsn.getBlockManager();
ExtendedBlock eb = util.getFirstBlock(dfs, path);
- DatanodeDescriptor dn = cluster.getNameNode().getNamesystem()
- .getBlockManager().getBlockCollection(eb.getLocalBlock())
- .getBlocks()[0].getDatanode(0);
- cluster.getNameNode().getNamesystem().getBlockManager()
- .getDatanodeManager().getDecomManager().startDecommission(dn);
+ long bcId = bm.getBlockCollectionId(eb.getLocalBlock());
+ DatanodeDescriptor dn = fsn.getBlockCollection(bcId).getBlocks()[0]
+ .getDatanode(0);
+ bm.getDatanodeManager().getDecomManager().startDecommission(dn);
String dnName = dn.getXferAddr();
// wait for decommission start
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
index 85072d1..711870d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
@@ -21,7 +21,6 @@ import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -41,6 +40,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.junit.After;
@@ -117,8 +117,7 @@ public class TestSnapshotBlocksMap {
static void assertBlockCollection(final BlockManager blkManager,
final INodeFile file, final BlockInfoContiguous b) {
Assert.assertSame(b, blkManager.getStoredBlock(b));
- Assert.assertSame(file, blkManager.getBlockCollection(b));
- Assert.assertSame(file, b.getBlockCollection());
+ Assert.assertEquals(file.getId(), b.getBlockCollectionId());
}
/**
@@ -150,7 +149,8 @@ public class TestSnapshotBlocksMap {
hdfs.delete(sub2, true);
// The INode should have been removed from the blocksMap
for(BlockInfoContiguous b : blocks) {
- assertNull(blockmanager.getBlockCollection(b));
+ assertEquals(INodeId.INVALID_INODE_ID,
+ blockmanager.getBlockCollectionId(b));
}
}
@@ -188,7 +188,7 @@ public class TestSnapshotBlocksMap {
hdfs.delete(file0, true);
// Make sure the blocks of file0 is still in blocksMap
for(BlockInfoContiguous b : blocks0) {
- assertNotNull(blockmanager.getBlockCollection(b));
+ assertTrue(INodeId.INVALID_INODE_ID != b.getBlockCollectionId());
}
assertBlockCollection(snapshotFile0.toString(), 4, fsdir, blockmanager);
@@ -202,7 +202,7 @@ public class TestSnapshotBlocksMap {
// Make sure the first block of file0 is still in blocksMap
for(BlockInfoContiguous b : blocks0) {
- assertNotNull(blockmanager.getBlockCollection(b));
+ assertTrue(INodeId.INVALID_INODE_ID != b.getBlockCollectionId());
}
assertBlockCollection(snapshotFile0.toString(), 4, fsdir, blockmanager);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1828b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index 07a0c38..beee5fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
@@ -268,7 +269,7 @@ public class TestSnapshotDeletion {
checkQuotaUsageComputation(dir, 8, BLOCKSIZE * REPLICATION * 3);
// check blocks of tempFile
for (BlockInfoContiguous b : blocks) {
- assertNull(blockmanager.getBlockCollection(b));
+ assertEquals(INodeId.INVALID_INODE_ID, b.getBlockCollectionId());
}
// make a change: create a new file under subsub
@@ -345,7 +346,7 @@ public class TestSnapshotDeletion {
// newFile
checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
for (BlockInfoContiguous b : blocks) {
- assertNull(blockmanager.getBlockCollection(b));
+ assertEquals(INodeId.INVALID_INODE_ID, b.getBlockCollectionId());
}
// make sure the whole subtree of sub is stored correctly in snapshot
@@ -508,7 +509,7 @@ public class TestSnapshotDeletion {
// metaChangeFile's replication factor decreases
checkQuotaUsageComputation(dir, 6, 2 * BLOCKSIZE * REPLICATION - BLOCKSIZE);
for (BlockInfoContiguous b : blocks) {
- assertNull(blockmanager.getBlockCollection(b));
+ assertEquals(INodeId.INVALID_INODE_ID, b.getBlockCollectionId());
}
// check 1. there is no snapshot s0
@@ -839,7 +840,7 @@ public class TestSnapshotDeletion {
assertFalse(hdfs.exists(file14_s1));
assertFalse(hdfs.exists(file15_s1));
for (BlockInfoContiguous b : blocks_14) {
- assertNull(blockmanager.getBlockCollection(b));
+ assertEquals(INodeId.INVALID_INODE_ID, b.getBlockCollectionId());
}
INodeFile nodeFile13 = (INodeFile) fsdir.getINode(file13.toString());