You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2011/06/29 01:43:05 UTC
svn commit: r1140909 [1/2] - in /hadoop/common/trunk/hdfs: ./
src/java/org/apache/hadoop/hdfs/server/balancer/
src/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/java/org/apache/hadoop/hdfs/server/common/
src/java/org/apache/hadoop/hdfs/server...
Author: szetszwo
Date: Tue Jun 28 23:43:03 2011
New Revision: 1140909
URL: http://svn.apache.org/viewvc?rev=1140909&view=rev
Log:
HDFS-2106. Move block management code from o.a.h.h.s.namenode to a new package o.a.h.h.s.blockmanagement.
Added:
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/PendingReplicationBlocks.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCorruptReplicaInfo.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDatanodeDescriptor.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestPendingReplication.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java
- copied, changed from r1140896, hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestUnderReplicatedBlocks.java
Removed:
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/PendingReplicationBlocks.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCorruptReplicaInfo.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDatanodeDescriptor.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestPendingReplication.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestUnderReplicatedBlocks.java
Modified:
hadoop/common/trunk/hdfs/CHANGES.txt
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DecommissionManager.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestComputeInvalidateWork.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHost2NodesMap.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
hadoop/common/trunk/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java
hadoop/common/trunk/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
hadoop/common/trunk/hdfs/src/webapps/hdfs/block_info_xml.jsp
Modified: hadoop/common/trunk/hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/CHANGES.txt?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hdfs/CHANGES.txt Tue Jun 28 23:43:03 2011
@@ -534,6 +534,9 @@ Trunk (unreleased changes)
HDFS-2110. StreamFile and ByteRangeInputStream cleanup. (eli)
+ HDFS-2106. Move block management code from o.a.h.h.s.namenode to a new
+ package o.a.h.h.s.blockmanagement. (szetszwo)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java Tue Jun 28 23:43:03 2011
@@ -61,10 +61,10 @@ import org.apache.hadoop.hdfs.protocol.d
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.common.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.Util;
-import org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicy;
-import org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.io.IOUtils;
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java Tue Jun 28 23:43:03 2011
@@ -15,16 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.util.LightWeightGSet;
/**
* Internal class for block metadata.
*/
-class BlockInfo extends Block implements LightWeightGSet.LinkedElement {
+public class BlockInfo extends Block implements LightWeightGSet.LinkedElement {
private INodeFile inode;
/** For implementing {@link LightWeightGSet.LinkedElement} interface */
@@ -44,12 +45,12 @@ class BlockInfo extends Block implements
* Construct an entry for blocksmap
* @param replication the block's replication factor
*/
- protected BlockInfo(int replication) {
+ public BlockInfo(int replication) {
this.triplets = new Object[3*replication];
this.inode = null;
}
- protected BlockInfo(Block blk, int replication) {
+ public BlockInfo(Block blk, int replication) {
super(blk);
this.triplets = new Object[3*replication];
this.inode = null;
@@ -65,11 +66,11 @@ class BlockInfo extends Block implements
this.inode = from.inode;
}
- INodeFile getINode() {
+ public INodeFile getINode() {
return inode;
}
- void setINode(INodeFile inode) {
+ public void setINode(INodeFile inode) {
this.inode = inode;
}
@@ -162,7 +163,7 @@ class BlockInfo extends Block implements
/**
* Add data-node this block belongs to.
*/
- boolean addNode(DatanodeDescriptor node) {
+ public boolean addNode(DatanodeDescriptor node) {
if(findDatanode(node) >= 0) // the node is already there
return false;
// find the last null node
@@ -176,7 +177,7 @@ class BlockInfo extends Block implements
/**
* Remove data-node from the block.
*/
- boolean removeNode(DatanodeDescriptor node) {
+ public boolean removeNode(DatanodeDescriptor node) {
int dnIndex = findDatanode(node);
if(dnIndex < 0) // the node is not found
return false;
@@ -218,7 +219,7 @@ class BlockInfo extends Block implements
* If the head is null then form a new list.
* @return current block as the new head of the list.
*/
- BlockInfo listInsert(BlockInfo head, DatanodeDescriptor dn) {
+ public BlockInfo listInsert(BlockInfo head, DatanodeDescriptor dn) {
int dnIndex = this.findDatanode(dn);
assert dnIndex >= 0 : "Data node is not found: current";
assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
@@ -238,7 +239,7 @@ class BlockInfo extends Block implements
* @return the new head of the list or null if the list becomes
* empty after deletion.
*/
- BlockInfo listRemove(BlockInfo head, DatanodeDescriptor dn) {
+ public BlockInfo listRemove(BlockInfo head, DatanodeDescriptor dn) {
if(head == null)
return null;
int dnIndex = this.findDatanode(dn);
@@ -284,7 +285,7 @@ class BlockInfo extends Block implements
* to {@link BlockInfoUnderConstruction}.
* @return {@link BlockUCState#COMPLETE}
*/
- BlockUCState getBlockUCState() {
+ public BlockUCState getBlockUCState() {
return BlockUCState.COMPLETE;
}
@@ -293,7 +294,7 @@ class BlockInfo extends Block implements
*
* @return true if the state of the block is {@link BlockUCState#COMPLETE}
*/
- boolean isComplete() {
+ public boolean isComplete() {
return getBlockUCState().equals(BlockUCState.COMPLETE);
}
@@ -302,7 +303,7 @@ class BlockInfo extends Block implements
*
* @return BlockInfoUnderConstruction - an under construction block.
*/
- BlockInfoUnderConstruction convertToBlockUnderConstruction(
+ public BlockInfoUnderConstruction convertToBlockUnderConstruction(
BlockUCState s, DatanodeDescriptor[] targets) {
if(isComplete()) {
return new BlockInfoUnderConstruction(
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java Tue Jun 28 23:43:03 2011
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.IOException;
import java.util.ArrayList;
@@ -24,12 +24,13 @@ import java.util.List;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
/**
* Represents a block that is currently being constructed.<br>
* This is usually the last block of a file opened for write or append.
*/
-class BlockInfoUnderConstruction extends BlockInfo {
+public class BlockInfoUnderConstruction extends BlockInfo {
/** Block state. See {@link BlockUCState} */
private BlockUCState blockUCState;
@@ -128,11 +129,14 @@ class BlockInfoUnderConstruction extends
* Create block and set its state to
* {@link BlockUCState#UNDER_CONSTRUCTION}.
*/
- BlockInfoUnderConstruction(Block blk, int replication) {
+ public BlockInfoUnderConstruction(Block blk, int replication) {
this(blk, replication, BlockUCState.UNDER_CONSTRUCTION, null);
}
- BlockInfoUnderConstruction(Block blk, int replication,
+ /**
+ * Create a block that is currently being constructed.
+ */
+ public BlockInfoUnderConstruction(Block blk, int replication,
BlockUCState state,
DatanodeDescriptor[] targets) {
super(blk, replication);
@@ -160,7 +164,8 @@ class BlockInfoUnderConstruction extends
return new BlockInfo(this);
}
- void setExpectedLocations(DatanodeDescriptor[] targets) {
+ /** Set expected locations */
+ public void setExpectedLocations(DatanodeDescriptor[] targets) {
int numLocations = targets == null ? 0 : targets.length;
this.replicas = new ArrayList<ReplicaUnderConstruction>(numLocations);
for(int i = 0; i < numLocations; i++)
@@ -172,7 +177,7 @@ class BlockInfoUnderConstruction extends
* Create array of expected replica locations
* (as has been assigned by chooseTargets()).
*/
- DatanodeDescriptor[] getExpectedLocations() {
+ public DatanodeDescriptor[] getExpectedLocations() {
int numLocations = replicas == null ? 0 : replicas.size();
DatanodeDescriptor[] locations = new DatanodeDescriptor[numLocations];
for(int i = 0; i < numLocations; i++)
@@ -180,7 +185,8 @@ class BlockInfoUnderConstruction extends
return locations;
}
- int getNumExpectedLocations() {
+ /** Get the number of expected locations */
+ public int getNumExpectedLocations() {
return replicas == null ? 0 : replicas.size();
}
@@ -189,7 +195,7 @@ class BlockInfoUnderConstruction extends
* @see BlockUCState
*/
@Override // BlockInfo
- BlockUCState getBlockUCState() {
+ public BlockUCState getBlockUCState() {
return blockUCState;
}
@@ -197,7 +203,8 @@ class BlockInfoUnderConstruction extends
blockUCState = s;
}
- long getBlockRecoveryId() {
+ /** Get block recovery ID */
+ public long getBlockRecoveryId() {
return blockRecoveryId;
}
@@ -220,7 +227,7 @@ class BlockInfoUnderConstruction extends
* Find the first alive data-node starting from the previous primary and
* make it primary.
*/
- void initializeBlockRecovery(long recoveryId) {
+ public void initializeBlockRecovery(long recoveryId) {
setBlockUCState(BlockUCState.UNDER_RECOVERY);
blockRecoveryId = recoveryId;
if (replicas.size() == 0) {
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Tue Jun 28 23:43:03 2011
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.IOException;
import java.io.PrintWriter;
@@ -39,10 +39,14 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportIterator;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks.BlockIterator;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NumberReplicas;
-import org.apache.hadoop.hdfs.server.namenode.UnderReplicatedBlocks;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
/**
* Keeps information related to the blocks stored in the Hadoop cluster.
@@ -57,18 +61,43 @@ public class BlockManager {
private final FSNamesystem namesystem;
- volatile long pendingReplicationBlocksCount = 0L;
- volatile long corruptReplicaBlocksCount = 0L;
- volatile long underReplicatedBlocksCount = 0L;
- volatile long scheduledReplicationBlocksCount = 0L;
- volatile long excessBlocksCount = 0L;
- volatile long pendingDeletionBlocksCount = 0L;
+ private volatile long pendingReplicationBlocksCount = 0L;
+ private volatile long corruptReplicaBlocksCount = 0L;
+ private volatile long underReplicatedBlocksCount = 0L;
+ public volatile long scheduledReplicationBlocksCount = 0L;
+ private volatile long excessBlocksCount = 0L;
+ private volatile long pendingDeletionBlocksCount = 0L;
- //
- // Mapping: Block -> { INode, datanodes, self ref }
- // Updated only in response to client-sent information.
- //
- final BlocksMap blocksMap;
+ /** Used by metrics */
+ public long getPendingReplicationBlocksCount() {
+ return pendingReplicationBlocksCount;
+ }
+ /** Used by metrics */
+ public long getUnderReplicatedBlocksCount() {
+ return underReplicatedBlocksCount;
+ }
+ /** Used by metrics */
+ public long getCorruptReplicaBlocksCount() {
+ return corruptReplicaBlocksCount;
+ }
+ /** Used by metrics */
+ public long getScheduledReplicationBlocksCount() {
+ return scheduledReplicationBlocksCount;
+ }
+ /** Used by metrics */
+ public long getPendingDeletionBlocksCount() {
+ return pendingDeletionBlocksCount;
+ }
+ /** Used by metrics */
+ public long getExcessBlocksCount() {
+ return excessBlocksCount;
+ }
+
+ /**
+ * Mapping: Block -> { INode, datanodes, self ref }
+ * Updated only in response to client-sent information.
+ */
+ public final BlocksMap blocksMap;
//
// Store blocks-->datanodedescriptor(s) map of corrupt replicas
@@ -90,24 +119,24 @@ public class BlockManager {
// eventually remove these extras.
// Mapping: StorageID -> TreeSet<Block>
//
- Map<String, Collection<Block>> excessReplicateMap =
+ public final Map<String, Collection<Block>> excessReplicateMap =
new TreeMap<String, Collection<Block>>();
//
// Store set of Blocks that need to be replicated 1 or more times.
// We also store pending replication-orders.
//
- UnderReplicatedBlocks neededReplications = new UnderReplicatedBlocks();
+ public UnderReplicatedBlocks neededReplications = new UnderReplicatedBlocks();
private PendingReplicationBlocks pendingReplications;
// The maximum number of replicas allowed for a block
- int maxReplication;
+ public int maxReplication;
// How many outgoing replication streams a given node should have at one time
- int maxReplicationStreams;
+ public int maxReplicationStreams;
// Minimum copies needed or else write is disallowed
- int minReplication;
+ public int minReplication;
// Default number of replicas
- int defaultReplication;
+ public int defaultReplication;
// How many entries are returned by getCorruptInodes()
int maxCorruptFilesReturned;
@@ -121,9 +150,9 @@ public class BlockManager {
Random r = new Random();
// for block replicas placement
- BlockPlacementPolicy replicator;
+ public BlockPlacementPolicy replicator;
- BlockManager(FSNamesystem fsn, Configuration conf) throws IOException {
+ public BlockManager(FSNamesystem fsn, Configuration conf) throws IOException {
this(fsn, conf, DEFAULT_INITIAL_MAP_CAPACITY);
}
@@ -178,16 +207,16 @@ public class BlockManager {
FSNamesystem.LOG.info("shouldCheckForEnoughRacks = " + shouldCheckForEnoughRacks);
}
- void activate() {
+ public void activate() {
pendingReplications.start();
}
- void close() {
+ public void close() {
if (pendingReplications != null) pendingReplications.stop();
blocksMap.close();
}
- void metaSave(PrintWriter out) {
+ public void metaSave(PrintWriter out) {
//
// Dump contents of neededReplication
//
@@ -249,7 +278,7 @@ public class BlockManager {
* @param block
* @return true if the block has minimum replicas
*/
- boolean checkMinReplication(Block block) {
+ public boolean checkMinReplication(Block block) {
return (countNodes(block).liveReplicas() >= minReplication);
}
@@ -297,7 +326,7 @@ public class BlockManager {
* @throws IOException if the block does not have at least a minimal number
* of replicas reported from data-nodes.
*/
- void commitOrCompleteLastBlock(INodeFileUnderConstruction fileINode,
+ public void commitOrCompleteLastBlock(INodeFileUnderConstruction fileINode,
Block commitBlock) throws IOException {
if(commitBlock == null)
@@ -362,7 +391,7 @@ public class BlockManager {
* @param fileINode file
* @return the last block locations if the block is partial or null otherwise
*/
- LocatedBlock convertLastBlockToUnderConstruction(
+ public LocatedBlock convertLastBlockToUnderConstruction(
INodeFileUnderConstruction fileINode) throws IOException {
BlockInfo oldBlock = fileINode.getLastBlock();
if(oldBlock == null ||
@@ -393,7 +422,7 @@ public class BlockManager {
/**
* Get all valid locations of the block
*/
- ArrayList<String> getValidLocations(Block block) {
+ public ArrayList<String> getValidLocations(Block block) {
ArrayList<String> machineSet =
new ArrayList<String>(blocksMap.numNodes(block));
for(Iterator<DatanodeDescriptor> it =
@@ -407,7 +436,7 @@ public class BlockManager {
return machineSet;
}
- List<LocatedBlock> getBlockLocations(BlockInfo[] blocks, long offset,
+ public List<LocatedBlock> getBlockLocations(BlockInfo[] blocks, long offset,
long length, int nrBlocksToReturn) throws IOException {
int curBlk = 0;
long curPos = 0, blkSize = 0;
@@ -436,11 +465,15 @@ public class BlockManager {
return results;
}
- /** @param needBlockToken
- * @return a LocatedBlock for the given block */
- LocatedBlock getBlockLocation(final BlockInfo blk, final long pos
+ /** @return a LocatedBlock for the given block */
+ public LocatedBlock getBlockLocation(final BlockInfo blk, final long pos
) throws IOException {
- if (!blk.isComplete()) {
+ if (blk instanceof BlockInfoUnderConstruction) {
+ if (blk.isComplete()) {
+ throw new IOException(
+ "blk instanceof BlockInfoUnderConstruction && blk.isComplete()"
+ + ", blk=" + blk);
+ }
final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)blk;
final DatanodeDescriptor[] locations = uc.getExpectedLocations();
return namesystem.createLocatedBlock(uc, locations, pos, false);
@@ -476,7 +509,7 @@ public class BlockManager {
* Check whether the replication parameter is within the range
* determined by system configuration.
*/
- void verifyReplication(String src,
+ public void verifyReplication(String src,
short replication,
String clientName) throws IOException {
@@ -544,7 +577,7 @@ public class BlockManager {
* @param b block
* @param dn datanode
*/
- void addToInvalidates(Block b, DatanodeInfo dn) {
+ public void addToInvalidates(Block b, DatanodeInfo dn) {
addToInvalidates(b, dn, true);
}
@@ -585,7 +618,7 @@ public class BlockManager {
}
}
- void findAndMarkBlockAsCorrupt(Block blk,
+ public void findAndMarkBlockAsCorrupt(Block blk,
DatanodeInfo dn) throws IOException {
BlockInfo storedBlock = getStoredBlock(blk);
if (storedBlock == null) {
@@ -668,14 +701,14 @@ public class BlockManager {
}
}
- void updateState() {
+ public void updateState() {
pendingReplicationBlocksCount = pendingReplications.size();
underReplicatedBlocksCount = neededReplications.size();
corruptReplicaBlocksCount = corruptReplicas.size();
}
/** Return number of under-replicated but not missing blocks */
- int getUnderReplicatedNotMissingBlocks() {
+ public int getUnderReplicatedNotMissingBlocks() {
return neededReplications.getUnderReplicatedBlockCount();
}
@@ -684,7 +717,7 @@ public class BlockManager {
* @param nodesToProcess number of datanodes to schedule deletion work
* @return total number of block for deletion
*/
- int computeInvalidateWork(int nodesToProcess) {
+ public int computeInvalidateWork(int nodesToProcess) {
int numOfNodes = recentInvalidateSets.size();
nodesToProcess = Math.min(numOfNodes, nodesToProcess);
@@ -724,7 +757,7 @@ public class BlockManager {
*
* @return number of blocks scheduled for replication during this iteration.
*/
- int computeReplicationWork(int blocksToProcess) throws IOException {
+ public int computeReplicationWork(int blocksToProcess) throws IOException {
// Choose the blocks to be replicated
List<List<Block>> blocksToReplicate =
chooseUnderReplicatedBlocks(blocksToProcess);
@@ -1031,7 +1064,7 @@ public class BlockManager {
* If there were any replication requests that timed out, reap them
* and put them back into the neededReplication queue
*/
- void processPendingReplications() {
+ public void processPendingReplications() {
Block[] timedOutItems = pendingReplications.getTimedOutBlocks();
if (timedOutItems != null) {
namesystem.writeLock();
@@ -1464,7 +1497,7 @@ public class BlockManager {
short fileReplication = fileINode.getReplication();
if (!isNeededReplication(storedBlock, fileReplication, numCurrentReplica)) {
neededReplications.remove(storedBlock, numCurrentReplica,
- num.decommissionedReplicas, fileReplication);
+ num.decommissionedReplicas(), fileReplication);
} else {
updateNeededReplications(storedBlock, curReplicaDelta, 0);
}
@@ -1525,7 +1558,7 @@ public class BlockManager {
* For each block in the name-node verify whether it belongs to any file,
* over or under replicated. Place it into the respective queue.
*/
- void processMisReplicatedBlocks() {
+ public void processMisReplicatedBlocks() {
long nrInvalid = 0, nrOverReplicated = 0, nrUnderReplicated = 0;
namesystem.writeLock();
try {
@@ -1570,7 +1603,7 @@ public class BlockManager {
* If there are any extras, call chooseExcessReplicates() to
* mark them in the excessReplicateMap.
*/
- void processOverReplicatedBlock(Block block, short replication,
+ public void processOverReplicatedBlock(Block block, short replication,
DatanodeDescriptor addedNode, DatanodeDescriptor delNodeHint) {
assert namesystem.hasWriteLock();
if (addedNode == delNodeHint) {
@@ -1597,7 +1630,7 @@ public class BlockManager {
addedNode, delNodeHint, replicator);
}
- void addToExcessReplicate(DatanodeInfo dn, Block block) {
+ public void addToExcessReplicate(DatanodeInfo dn, Block block) {
assert namesystem.hasWriteLock();
Collection<Block> excessBlocks = excessReplicateMap.get(dn.getStorageID());
if (excessBlocks == null) {
@@ -1618,7 +1651,7 @@ public class BlockManager {
* Modify (block-->datanode) map. Possibly generate replication tasks, if the
* removed block is still valid.
*/
- void removeStoredBlock(Block block, DatanodeDescriptor node) {
+ public void removeStoredBlock(Block block, DatanodeDescriptor node) {
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* NameSystem.removeStoredBlock: "
+ block + " from " + node.getName());
@@ -1673,7 +1706,7 @@ public class BlockManager {
/**
* The given node is reporting that it received a certain block.
*/
- void addBlock(DatanodeDescriptor node, Block block, String delHint)
+ public void addBlock(DatanodeDescriptor node, Block block, String delHint)
throws IOException {
// decrement number of blocks scheduled to this datanode.
node.decBlocksScheduled();
@@ -1726,7 +1759,7 @@ public class BlockManager {
/**
* Return the number of nodes that are live and decommissioned.
*/
- NumberReplicas countNodes(Block b) {
+ public NumberReplicas countNodes(Block b) {
int count = 0;
int live = 0;
int corrupt = 0;
@@ -1805,7 +1838,7 @@ public class BlockManager {
* Return true if there are any blocks on this node that have not
* yet reached their replication factor. Otherwise returns false.
*/
- boolean isReplicationInProgress(DatanodeDescriptor srcNode) {
+ public boolean isReplicationInProgress(DatanodeDescriptor srcNode) {
boolean status = false;
int underReplicatedBlocks = 0;
int decommissionOnlyReplicas = 0;
@@ -1855,11 +1888,11 @@ public class BlockManager {
return status;
}
- int getActiveBlockCount() {
+ public int getActiveBlockCount() {
return blocksMap.size() - (int)pendingDeletionBlocksCount;
}
- DatanodeDescriptor[] getNodes(BlockInfo block) {
+ public DatanodeDescriptor[] getNodes(BlockInfo block) {
DatanodeDescriptor[] nodes =
new DatanodeDescriptor[block.numNodes()];
Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block);
@@ -1869,22 +1902,22 @@ public class BlockManager {
return nodes;
}
- int getTotalBlocks() {
+ public int getTotalBlocks() {
return blocksMap.size();
}
- void removeBlock(Block block) {
+ public void removeBlock(Block block) {
addToInvalidates(block);
corruptReplicas.removeFromCorruptReplicasMap(block);
blocksMap.removeBlock(block);
}
- BlockInfo getStoredBlock(Block block) {
+ public BlockInfo getStoredBlock(Block block) {
return blocksMap.getStoredBlock(block);
}
/* updates a block in under replication queue */
- void updateNeededReplications(Block block, int curReplicasDelta,
+ public void updateNeededReplications(Block block, int curReplicasDelta,
int expectedReplicasDelta) {
namesystem.writeLock();
try {
@@ -1905,13 +1938,13 @@ public class BlockManager {
}
}
- void checkReplication(Block block, int numExpectedReplicas) {
+ public void checkReplication(Block block, int numExpectedReplicas) {
// filter out containingNodes that are marked for decommission.
NumberReplicas number = countNodes(block);
if (isNeededReplication(block, numExpectedReplicas, number.liveReplicas())) {
neededReplications.add(block,
number.liveReplicas(),
- number.decommissionedReplicas,
+ number.decommissionedReplicas(),
numExpectedReplicas);
}
}
@@ -1926,11 +1959,8 @@ public class BlockManager {
return fileINode.getReplication();
}
- /**
- * Remove a datanode from the invalidatesSet
- * @param n datanode
- */
- void removeFromInvalidates(String storageID) {
+ /** Remove a datanode from the invalidatesSet */
+ public void removeFromInvalidates(String storageID) {
Collection<Block> blocks = recentInvalidateSets.remove(storageID);
if (blocks != null) {
pendingDeletionBlocksCount -= blocks.size();
@@ -1998,7 +2028,7 @@ public class BlockManager {
//Returns the number of racks over which a given block is replicated
//decommissioning/decommissioned nodes are not counted. corrupt replicas
//are also ignored
- int getNumberOfRacks(Block b) {
+ public int getNumberOfRacks(Block b) {
HashSet<String> rackSet = new HashSet<String>(0);
Collection<DatanodeDescriptor> corruptNodes =
corruptReplicas.getNodes(b);
@@ -2056,32 +2086,32 @@ public class BlockManager {
}
}
- long getMissingBlocksCount() {
+ public long getMissingBlocksCount() {
// not locking
return this.neededReplications.getCorruptBlockSize();
}
- BlockInfo addINode(BlockInfo block, INodeFile iNode) {
+ public BlockInfo addINode(BlockInfo block, INodeFile iNode) {
return blocksMap.addINode(block, iNode);
}
- INodeFile getINode(Block b) {
+ public INodeFile getINode(Block b) {
return blocksMap.getINode(b);
}
- void removeFromCorruptReplicasMap(Block block) {
+ public void removeFromCorruptReplicasMap(Block block) {
corruptReplicas.removeFromCorruptReplicasMap(block);
}
- int numCorruptReplicas(Block block) {
+ public int numCorruptReplicas(Block block) {
return corruptReplicas.numCorruptReplicas(block);
}
- void removeBlockFromMap(Block block) {
+ public void removeBlockFromMap(Block block) {
blocksMap.removeBlock(block);
}
- int getCapacity() {
+ public int getCapacity() {
namesystem.readLock();
try {
return blocksMap.getCapacity();
@@ -2104,7 +2134,7 @@ public class BlockManager {
* @return Up to numExpectedBlocks blocks from startingBlockId if it exists
*
*/
- long[] getCorruptReplicaBlockIds(int numExpectedBlocks,
+ public long[] getCorruptReplicaBlockIds(int numExpectedBlocks,
Long startingBlockId) {
return corruptReplicas.getCorruptReplicaBlockIds(numExpectedBlocks,
startingBlockId);
@@ -2113,7 +2143,7 @@ public class BlockManager {
/**
* Return an iterator over the set of blocks for which there are no replicas.
*/
- UnderReplicatedBlocks.BlockIterator getCorruptReplicaBlockIterator() {
+ public BlockIterator getCorruptReplicaBlockIterator() {
return neededReplications
.iterator(UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
}
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java Tue Jun 28 23:43:03 2011
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.ArrayList;
import java.util.Collection;
@@ -26,6 +26,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.namenode.FSClusterStats;
+import org.apache.hadoop.hdfs.server.namenode.FSInodeInfo;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.util.ReflectionUtils;
@@ -94,7 +96,7 @@ public abstract class BlockPlacementPoli
* @return array of DatanodeDescriptor instances chosen as target
* and sorted as a pipeline.
*/
- abstract DatanodeDescriptor[] chooseTarget(String srcPath,
+ public abstract DatanodeDescriptor[] chooseTarget(String srcPath,
int numOfReplicas,
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
@@ -222,11 +224,11 @@ public abstract class BlockPlacementPoli
* @param numOfReplicas number of replicas wanted.
* @param writer the writer's machine, null if not in the cluster.
* @param blocksize size of the data to be written.
- * @param excludedNodes: datanodes that should not be considered as targets.
+ * @param excludedNodes datanodes that should not be considered as targets.
* @return array of DatanodeDescriptor instances chosen as targets
* and sorted as a pipeline.
*/
- DatanodeDescriptor[] chooseTarget(String srcPath,
+ public DatanodeDescriptor[] chooseTarget(String srcPath,
int numOfReplicas,
DatanodeDescriptor writer,
HashMap<Node, Node> excludedNodes,
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Tue Jun 28 23:43:03 2011
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.ArrayList;
import java.util.Collection;
@@ -32,6 +32,9 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.namenode.FSClusterStats;
+import org.apache.hadoop.hdfs.server.namenode.FSInodeInfo;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
@@ -89,7 +92,7 @@ public class BlockPlacementPolicyDefault
/** {@inheritDoc} */
@Override
- DatanodeDescriptor[] chooseTarget(String srcPath,
+ public DatanodeDescriptor[] chooseTarget(String srcPath,
int numOfReplicas,
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java Tue Jun 28 23:43:03 2011
@@ -15,11 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.Iterator;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.util.GSet;
import org.apache.hadoop.hdfs.util.LightWeightGSet;
@@ -28,7 +29,7 @@ import org.apache.hadoop.hdfs.util.Light
* block's metadata currently includes INode it belongs to and
* the datanodes that store the block.
*/
-class BlocksMap {
+public class BlocksMap {
private static class NodeIterator implements Iterator<DatanodeDescriptor> {
private BlockInfo blockInfo;
private int nextIdx = 0;
@@ -100,7 +101,7 @@ class BlocksMap {
/**
* Add block b belonging to the specified file inode to the map.
*/
- BlockInfo addINode(BlockInfo b, INodeFile iNode) {
+ public BlockInfo addINode(BlockInfo b, INodeFile iNode) {
BlockInfo info = blocks.get(b);
if (info != b) {
info = b;
@@ -136,7 +137,7 @@ class BlocksMap {
* Searches for the block in the BlocksMap and
* returns Iterator that iterates through the nodes the block belongs to.
*/
- Iterator<DatanodeDescriptor> nodeIterator(Block b) {
+ public Iterator<DatanodeDescriptor> nodeIterator(Block b) {
return nodeIterator(blocks.get(b));
}
@@ -185,7 +186,7 @@ class BlocksMap {
/**
* Check if the block exists in map
*/
- boolean contains(Block block) {
+ public boolean contains(Block block) {
return blocks.contains(block);
}
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/CorruptReplicasMap.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java Tue Jun 28 23:43:03 2011
@@ -15,10 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.Server;
import java.util.*;
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java Tue Jun 28 23:43:03 2011
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.DataInput;
import java.io.IOException;
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.io.WritableUtils;
@@ -44,7 +45,7 @@ public class DatanodeDescriptor extends
// Stores status of decommissioning.
// If node is not decommissioning, do not use this object for anything.
- DecommissioningStatus decommissioningStatus = new DecommissioningStatus();
+ public DecommissioningStatus decommissioningStatus = new DecommissioningStatus();
/** Block and targets pair */
@InterfaceAudience.Private
@@ -96,8 +97,8 @@ public class DatanodeDescriptor extends
private int numBlocks = 0;
// isAlive == heartbeats.contains(this)
// This is an optimization, because contains takes O(n) time on Arraylist
- protected boolean isAlive = false;
- protected boolean needKeyUpdate = false;
+ public boolean isAlive = false;
+ public boolean needKeyUpdate = false;
/** A queue of blocks to be replicated by this datanode */
private BlockQueue<BlockTargetPair> replicateBlocks = new BlockQueue<BlockTargetPair>();
@@ -204,7 +205,7 @@ public class DatanodeDescriptor extends
* Add datanode to the block.
* Add block to the head of the list of blocks belonging to the data-node.
*/
- boolean addBlock(BlockInfo b) {
+ public boolean addBlock(BlockInfo b) {
if(!b.addNode(this))
return false;
// add to the head of the data-node list
@@ -217,7 +218,7 @@ public class DatanodeDescriptor extends
* Remove block from the list of blocks belonging to the data-node.
* Remove datanode from the block.
*/
- boolean removeBlock(BlockInfo b) {
+ public boolean removeBlock(BlockInfo b) {
blockList = b.listRemove(blockList, this);
if ( b.removeNode(this) ) {
numBlocks--;
@@ -242,7 +243,7 @@ public class DatanodeDescriptor extends
* @param newBlock - a replacement block
* @return the new block
*/
- BlockInfo replaceBlock(BlockInfo oldBlock, BlockInfo newBlock) {
+ public BlockInfo replaceBlock(BlockInfo oldBlock, BlockInfo newBlock) {
boolean done = removeBlock(oldBlock);
assert done : "Old block should belong to the data-node when replacing";
done = addBlock(newBlock);
@@ -250,7 +251,7 @@ public class DatanodeDescriptor extends
return newBlock;
}
- void resetBlocks() {
+ public void resetBlocks() {
this.capacity = 0;
this.remaining = 0;
this.blockPoolUsed = 0;
@@ -268,7 +269,7 @@ public class DatanodeDescriptor extends
/**
* Updates stats from datanode heartbeat.
*/
- void updateHeartbeat(long capacity, long dfsUsed, long remaining,
+ public void updateHeartbeat(long capacity, long dfsUsed, long remaining,
long blockPoolUsed, int xceiverCount, int volFailures) {
this.capacity = capacity;
this.dfsUsed = dfsUsed;
@@ -283,7 +284,7 @@ public class DatanodeDescriptor extends
/**
* Iterates over the list of blocks belonging to the datanode.
*/
- static class BlockIterator implements Iterator<BlockInfo> {
+ public static class BlockIterator implements Iterator<BlockInfo> {
private BlockInfo current;
private DatanodeDescriptor node;
@@ -307,7 +308,7 @@ public class DatanodeDescriptor extends
}
}
- Iterator<BlockInfo> getBlockIterator() {
+ public Iterator<BlockInfo> getBlockIterator() {
return new BlockIterator(this.blockList, this);
}
@@ -361,11 +362,11 @@ public class DatanodeDescriptor extends
}
}
- List<BlockTargetPair> getReplicationCommand(int maxTransfers) {
+ public List<BlockTargetPair> getReplicationCommand(int maxTransfers) {
return replicateBlocks.poll(maxTransfers);
}
- BlockInfoUnderConstruction[] getLeaseRecoveryCommand(int maxTransfers) {
+ public BlockInfoUnderConstruction[] getLeaseRecoveryCommand(int maxTransfers) {
List<BlockInfoUnderConstruction> blocks = recoverBlocks.poll(maxTransfers);
if(blocks == null)
return null;
@@ -375,7 +376,7 @@ public class DatanodeDescriptor extends
/**
* Remove the specified number of blocks to be invalidated
*/
- Block[] getInvalidateBlocks(int maxblocks) {
+ public Block[] getInvalidateBlocks(int maxblocks) {
return getBlockArray(invalidateBlocks, maxblocks);
}
@@ -418,7 +419,7 @@ public class DatanodeDescriptor extends
}
/** Serialization for FSEditLog */
- void readFieldsFromFSEditLog(DataInput in) throws IOException {
+ public void readFieldsFromFSEditLog(DataInput in) throws IOException {
this.name = DeprecatedUTF8.readString(in);
this.storageID = DeprecatedUTF8.readString(in);
this.infoPort = in.readShort() & 0x0000ffff;
@@ -445,7 +446,7 @@ public class DatanodeDescriptor extends
/**
* Increments counter for number of blocks scheduled.
*/
- void incBlocksScheduled() {
+ public void incBlocksScheduled() {
currApproxBlocksScheduled++;
}
@@ -485,12 +486,13 @@ public class DatanodeDescriptor extends
// by DatanodeID
return (this == obj) || super.equals(obj);
}
-
- class DecommissioningStatus {
- int underReplicatedBlocks;
- int decommissionOnlyReplicas;
- int underReplicatedInOpenFiles;
- long startTime;
+
+ /** Decommissioning status */
+ public class DecommissioningStatus {
+ private int underReplicatedBlocks;
+ private int decommissionOnlyReplicas;
+ private int underReplicatedInOpenFiles;
+ private long startTime;
synchronized void set(int underRep,
int onlyRep, int underConstruction) {
@@ -501,32 +503,34 @@ public class DatanodeDescriptor extends
decommissionOnlyReplicas = onlyRep;
underReplicatedInOpenFiles = underConstruction;
}
-
- synchronized int getUnderReplicatedBlocks() {
+
+ /** @return the number of under-replicated blocks */
+ public synchronized int getUnderReplicatedBlocks() {
if (isDecommissionInProgress() == false) {
return 0;
}
return underReplicatedBlocks;
}
- synchronized int getDecommissionOnlyReplicas() {
+ /** @return the number of decommission-only replicas */
+ public synchronized int getDecommissionOnlyReplicas() {
if (isDecommissionInProgress() == false) {
return 0;
}
return decommissionOnlyReplicas;
}
-
- synchronized int getUnderReplicatedInOpenFiles() {
+ /** @return the number of under-replicated blocks in open files */
+ public synchronized int getUnderReplicatedInOpenFiles() {
if (isDecommissionInProgress() == false) {
return 0;
}
return underReplicatedInOpenFiles;
}
-
- synchronized void setStartTime(long time) {
+ /** Set start time */
+ public synchronized void setStartTime(long time) {
startTime = time;
}
-
- synchronized long getStartTime() {
+ /** @return start time */
+ public synchronized long getStartTime() {
if (isDecommissionInProgress() == false) {
return 0;
}
@@ -538,11 +542,11 @@ public class DatanodeDescriptor extends
* Set the flag to indicate if this datanode is disallowed from communicating
* with the namenode.
*/
- void setDisallowed(boolean flag) {
+ public void setDisallowed(boolean flag) {
disallowed = flag;
}
-
- boolean isDisallowed() {
+ /** Is the datanode disallowed from communicating with the namenode? */
+ public boolean isDisallowed() {
return disallowed;
}
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/PendingReplicationBlocks.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/PendingReplicationBlocks.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/PendingReplicationBlocks.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java Tue Jun 28 23:43:03 2011
@@ -15,9 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+
import static org.apache.hadoop.hdfs.server.common.Util.now;
import org.apache.hadoop.util.*;
import java.io.*;
Copied: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java (from r1140896, hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java)
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java?p2=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java&p1=hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java&r1=1140896&r2=1140909&rev=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/UnderReplicatedBlocks.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java Tue Jun 28 23:43:03 2011
@@ -15,17 +15,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.namenode;
+package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.*;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
/* Class for keeping track of under replication blocks
* Blocks have replication priority, with priority 0 indicating the highest
* Blocks have only one replicas has the highest
*/
-class UnderReplicatedBlocks implements Iterable<Block> {
+public class UnderReplicatedBlocks implements Iterable<Block> {
static final int LEVEL = 5;
static public final int QUEUE_WITH_CORRUPT_BLOCKS = 4;
private List<TreeSet<Block>> priorityQueues = new ArrayList<TreeSet<Block>>();
@@ -47,7 +48,7 @@ class UnderReplicatedBlocks implements I
}
/* Return the total number of under replication blocks */
- synchronized int size() {
+ public synchronized int size() {
int size = 0;
for (int i=0; i<LEVEL; i++) {
size += priorityQueues.get(i).size();
@@ -70,7 +71,7 @@ class UnderReplicatedBlocks implements I
}
/* Check if a block is in the neededReplication queue */
- synchronized boolean contains(Block block) {
+ public synchronized boolean contains(Block block) {
for(TreeSet<Block> set:priorityQueues) {
if(set.contains(block)) { return true; }
}
@@ -218,7 +219,7 @@ class UnderReplicatedBlocks implements I
return new BlockIterator();
}
- class BlockIterator implements Iterator<Block> {
+ public class BlockIterator implements Iterator<Block> {
private int level;
private boolean isIteratorForLevel = false;
private List<Iterator<Block>> iterators = new ArrayList<Iterator<Block>>();
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java Tue Jun 28 23:43:03 2011
@@ -26,8 +26,8 @@ import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL;
import java.net.URLEncoder;
-import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -45,14 +45,13 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.BlockReader;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.http.HtmlQuoting;
import org.apache.hadoop.io.Text;
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DecommissionManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DecommissionManager.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DecommissionManager.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/DecommissionManager.java Tue Jun 28 23:43:03 2011
@@ -21,6 +21,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.util.CyclicIteration;
/**
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Tue Jun 28 23:43:03 2011
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.server.common.Util.now;
+
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -24,35 +26,40 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.Condition;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.ParentNotDirectoryException;
-import org.apache.hadoop.fs.UnresolvedLinkException;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.FSConstants;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.FSLimitException;
-import org.apache.hadoop.hdfs.protocol.FSLimitException.*;
+import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
+import org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.hdfs.util.ByteArray;
-import static org.apache.hadoop.hdfs.server.common.Util.now;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
/*************************************************
* FSDirectory stores the filesystem directory state.
@@ -63,7 +70,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
* and logged to disk.
*
*************************************************/
-class FSDirectory implements Closeable {
+public class FSDirectory implements Closeable {
INodeDirectoryWithQuota rootDir;
FSImage fsImage;
@@ -1332,7 +1339,7 @@ class FSDirectory implements Closeable {
* @throws QuotaExceededException if the new count violates any quota limit
* @throws FileNotFound if path does not exist.
*/
- void updateSpaceConsumed(String path, long nsDelta, long dsDelta)
+ public void updateSpaceConsumed(String path, long nsDelta, long dsDelta)
throws QuotaExceededException,
FileNotFoundException,
UnresolvedLinkException {
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Tue Jun 28 23:43:03 2011
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.server.common.Util.now;
+
import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.EOFException;
@@ -32,10 +34,30 @@ import org.apache.hadoop.hdfs.protocol.F
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
-import static org.apache.hadoop.hdfs.server.common.Util.now;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddCloseOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.CancelDelegationTokenOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ClearNSQuotaOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ConcatDeleteOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.DeleteOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.GetDelegationTokenOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ReassignLeaseOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOldOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenewDelegationTokenOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetGenstampOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetNSQuotaOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetOwnerOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetPermissionsOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetQuotaOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetReplicationOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SymlinkOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateMasterKeyOp;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.*;
public class FSEditLogLoader {
private final FSNamesystem fsNamesys;
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java Tue Jun 28 23:43:03 2011
@@ -32,6 +32,8 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.*;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.io.BytesWritable;
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Tue Jun 28 23:43:03 2011
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java Tue Jun 28 23:43:03 2011
@@ -22,7 +22,6 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.nio.ByteBuffer;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -33,6 +32,9 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Jun 28 23:43:03 2011
@@ -97,15 +97,20 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.common.Util;
-import org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
-import org.apache.hadoop.hdfs.server.namenode.UnderReplicatedBlocks.BlockIterator;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
@@ -235,7 +240,7 @@ public class FSNamesystem implements FSC
// Stores the correct file name hierarchy
//
public FSDirectory dir;
- BlockManager blockManager;
+ public BlockManager blockManager;
// Block pool ID used by this namenode
String blockPoolId;
@@ -270,10 +275,10 @@ public class FSNamesystem implements FSC
* Stores a set of DatanodeDescriptor objects.
* This is a subset of {@link #datanodeMap}, containing nodes that are
* considered alive.
- * The {@link HeartbeatMonitor} periodically checks for outdated entries,
+ * The HeartbeatMonitor periodically checks for out-dated entries,
* and removes them from the list.
*/
- ArrayList<DatanodeDescriptor> heartbeats = new ArrayList<DatanodeDescriptor>();
+ public ArrayList<DatanodeDescriptor> heartbeats = new ArrayList<DatanodeDescriptor>();
public LeaseManager leaseManager = new LeaseManager(this);
@@ -314,8 +319,8 @@ public class FSNamesystem implements FSC
private volatile SafeModeInfo safeMode; // safe mode information
private Host2NodesMap host2DataNodeMap = new Host2NodesMap();
- // datanode networktoplogy
- NetworkTopology clusterMap = new NetworkTopology();
+ /** datanode network toplogy */
+ public NetworkTopology clusterMap = new NetworkTopology();
private DNSToSwitchMapping dnsToSwitchMapping;
private HostsFileReader hostsReader;
@@ -329,7 +334,7 @@ public class FSNamesystem implements FSC
private final GenerationStamp generationStamp = new GenerationStamp();
// Ask Datanode only up to this many blocks to delete.
- int blockInvalidateLimit = DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT;
+ public int blockInvalidateLimit = DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT;
// precision of access times.
private long accessTimePrecision = 0;
@@ -472,23 +477,23 @@ public class FSNamesystem implements FSC
}
// utility methods to acquire and release read lock and write lock
- void readLock() {
+ public void readLock() {
this.fsLock.readLock().lock();
}
- void readUnlock() {
+ public void readUnlock() {
this.fsLock.readLock().unlock();
}
- void writeLock() {
+ public void writeLock() {
this.fsLock.writeLock().lock();
}
- void writeUnlock() {
+ public void writeUnlock() {
this.fsLock.writeLock().unlock();
}
- boolean hasWriteLock() {
+ public boolean hasWriteLock() {
return this.fsLock.isWriteLockedByCurrentThread();
}
@@ -1014,7 +1019,7 @@ public class FSNamesystem implements FSC
}
/** Create a LocatedBlock. */
- LocatedBlock createLocatedBlock(final Block b, final DatanodeInfo[] locations,
+ public LocatedBlock createLocatedBlock(final Block b, final DatanodeInfo[] locations,
final long offset, final boolean corrupt) throws IOException {
return new LocatedBlock(getExtendedBlock(b), locations, offset, corrupt);
}
@@ -3013,7 +3018,7 @@ public class FSNamesystem implements FSC
* @return an array of datanode commands
* @throws IOException
*/
- DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
+ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
long capacity, long dfsUsed, long remaining, long blockPoolUsed,
int xceiverCount, int xmitsInProgress, int failedVolumes)
throws IOException {
@@ -3521,7 +3526,7 @@ public class FSNamesystem implements FSC
* If no such a node is available,
* then pick a node with least free space
*/
- void chooseExcessReplicates(Collection<DatanodeDescriptor> nonExcess,
+ public void chooseExcessReplicates(Collection<DatanodeDescriptor> nonExcess,
Block b, short replication,
DatanodeDescriptor addedNode,
DatanodeDescriptor delNodeHint,
@@ -3980,45 +3985,6 @@ public class FSNamesystem implements FSC
}
/**
- * A immutable object that stores the number of live replicas and
- * the number of decommissined Replicas.
- */
- static class NumberReplicas {
- private int liveReplicas;
- int decommissionedReplicas;
- private int corruptReplicas;
- private int excessReplicas;
-
- NumberReplicas() {
- initialize(0, 0, 0, 0);
- }
-
- NumberReplicas(int live, int decommissioned, int corrupt, int excess) {
- initialize(live, decommissioned, corrupt, excess);
- }
-
- void initialize(int live, int decommissioned, int corrupt, int excess) {
- liveReplicas = live;
- decommissionedReplicas = decommissioned;
- corruptReplicas = corrupt;
- excessReplicas = excess;
- }
-
- int liveReplicas() {
- return liveReplicas;
- }
- int decommissionedReplicas() {
- return decommissionedReplicas;
- }
- int corruptReplicas() {
- return corruptReplicas;
- }
- int excessReplicas() {
- return excessReplicas;
- }
- }
-
- /**
* Change, if appropriate, the admin state of a datanode to
* decommission completed. Return true if decommission is complete.
*/
@@ -4675,7 +4641,7 @@ public class FSNamesystem implements FSC
* Check whether the name node is in safe mode.
* @return true if safe mode is ON, false otherwise
*/
- boolean isInSafeMode() {
+ public boolean isInSafeMode() {
// safeMode is volatile, and may be set to null at any time
SafeModeInfo safeMode = this.safeMode;
if (safeMode == null)
@@ -4686,7 +4652,7 @@ public class FSNamesystem implements FSC
/**
* Check whether the name node is in startup mode.
*/
- boolean isInStartupSafeMode() {
+ public boolean isInStartupSafeMode() {
// safeMode is volatile, and may be set to null at any time
SafeModeInfo safeMode = this.safeMode;
if (safeMode == null)
@@ -4697,7 +4663,7 @@ public class FSNamesystem implements FSC
/**
* Check whether replication queues are populated.
*/
- boolean isPopulatingReplQueues() {
+ public boolean isPopulatingReplQueues() {
// safeMode is volatile, and may be set to null at any time
SafeModeInfo safeMode = this.safeMode;
if (safeMode == null)
@@ -4709,7 +4675,7 @@ public class FSNamesystem implements FSC
* Increment number of blocks that reached minimal replication.
* @param replication current replication
*/
- void incrementSafeBlockCount(int replication) {
+ public void incrementSafeBlockCount(int replication) {
// safeMode is volatile, and may be set to null at any time
SafeModeInfo safeMode = this.safeMode;
if (safeMode == null)
@@ -4720,7 +4686,7 @@ public class FSNamesystem implements FSC
/**
* Decrement number of blocks that reached minimal replication.
*/
- void decrementSafeBlockCount(Block b) {
+ public void decrementSafeBlockCount(Block b) {
// safeMode is volatile, and may be set to null at any time
SafeModeInfo safeMode = this.safeMode;
if (safeMode == null) // mostly true
@@ -5042,13 +5008,13 @@ public class FSNamesystem implements FSC
@Override // FSNamesystemMBean
@Metric
public long getPendingReplicationBlocks() {
- return blockManager.pendingReplicationBlocksCount;
+ return blockManager.getPendingReplicationBlocksCount();
}
@Override // FSNamesystemMBean
@Metric
public long getUnderReplicatedBlocks() {
- return blockManager.underReplicatedBlocksCount;
+ return blockManager.getUnderReplicatedBlocksCount();
}
/** Return number of under-replicated but not missing blocks */
@@ -5059,23 +5025,23 @@ public class FSNamesystem implements FSC
/** Returns number of blocks with corrupt replicas */
@Metric({"CorruptBlocks", "Number of blocks with corrupt replicas"})
public long getCorruptReplicaBlocks() {
- return blockManager.corruptReplicaBlocksCount;
+ return blockManager.getCorruptReplicaBlocksCount();
}
@Override // FSNamesystemMBean
@Metric
public long getScheduledReplicationBlocks() {
- return blockManager.scheduledReplicationBlocksCount;
+ return blockManager.getScheduledReplicationBlocksCount();
}
@Metric
public long getPendingDeletionBlocks() {
- return blockManager.pendingDeletionBlocksCount;
+ return blockManager.getPendingDeletionBlocksCount();
}
@Metric
public long getExcessBlocks() {
- return blockManager.excessBlocksCount;
+ return blockManager.getExcessBlocksCount();
}
@Metric
@@ -5444,7 +5410,7 @@ public class FSNamesystem implements FSC
}
/** Get a datanode descriptor given corresponding storageID */
- DatanodeDescriptor getDatanode(String nodeID) {
+ public DatanodeDescriptor getDatanode(String nodeID) {
assert hasReadOrWriteLock();
return datanodeMap.get(nodeID);
}
@@ -5508,7 +5474,7 @@ public class FSNamesystem implements FSC
if (startBlockAfter != null) {
startBlockId = Block.filename2id(startBlockAfter);
}
- BlockIterator blkIterator = blockManager.getCorruptReplicaBlockIterator();
+ UnderReplicatedBlocks.BlockIterator blkIterator = blockManager.getCorruptReplicaBlockIterator();
while (blkIterator.hasNext()) {
Block blk = blkIterator.next();
INode inode = blockManager.getINode(blk);
Modified: hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java?rev=1140909&r1=1140908&r2=1140909&view=diff
==============================================================================
--- hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java (original)
+++ hadoop/common/trunk/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/Host2NodesMap.java Tue Jun 28 23:43:03 2011
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.util.*;
+import java.util.HashMap;
+import java.util.Random;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+
class Host2NodesMap {
private HashMap<String, DatanodeDescriptor[]> map
= new HashMap<String, DatanodeDescriptor[]>();