You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2007/03/17 21:18:25 UTC
svn commit: r519437 - in /lucene/hadoop/trunk: CHANGES.txt
src/java/org/apache/hadoop/dfs/FSNamesystem.java
Author: tomwhite
Date: Sat Mar 17 13:18:24 2007
New Revision: 519437
URL: http://svn.apache.org/viewvc?view=rev&rev=519437
Log:
HADOOP-1126. Optimize CPU usage for under replicated blocks when cluster restarts. Contributed by Hairong Kuang.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=519437&r1=519436&r2=519437
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Sat Mar 17 13:18:24 2007
@@ -76,6 +76,9 @@
22. HADOOP-1129. Fix DFSClient to not hide IOExceptions in
flush method. (Hairong Kuang via tomwhite)
+23. HADOOP-1126. Optimize CPU usage for under replicated blocks
+ when cluster restarts. (Hairong Kuang via tomwhite)
+
Release 0.12.0 - 2007-03-02
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=519437&r1=519436&r2=519437
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Sat Mar 17 13:18:24 2007
@@ -155,7 +155,7 @@
// We also store pending replication-orders.
// Set of: Block
//
- private UnderReplicationBlocks neededReplications = new UnderReplicationBlocks();
+ private UnderReplicatedBlocks neededReplications = new UnderReplicatedBlocks();
private PendingReplicationBlocks pendingReplications;
//
@@ -334,12 +334,12 @@
* Blocks have replication priority, with priority 0 indicating the highest
* Blocks have only one replicas has the highest
*/
- private class UnderReplicationBlocks {
+ private class UnderReplicatedBlocks {
private static final int LEVEL = 3;
TreeSet<Block>[] priorityQueues = new TreeSet[LEVEL];
/* constructor */
- UnderReplicationBlocks() {
+ UnderReplicatedBlocks() {
for(int i=0; i<LEVEL; i++) {
priorityQueues[i] = new TreeSet<Block>();
}
@@ -369,7 +369,7 @@
*/
private int getPriority(Block block,
int curReplicas, int expectedReplicas) {
- if (curReplicas==0 || curReplicas>=expectedReplicas) {
+ if (curReplicas<=0 || curReplicas>=expectedReplicas) {
return LEVEL; // no need to replicate
} else if(curReplicas==1) {
return 0; // highest priority
@@ -414,16 +414,14 @@
/* remove a block from a under replication queue */
synchronized boolean remove(Block block,
int oldReplicas, int oldExpectedReplicas) {
- if(oldExpectedReplicas <= oldReplicas) {
- return false;
- }
int priLevel = getPriority(block, oldReplicas, oldExpectedReplicas);
return remove(block, priLevel);
}
/* remove a block from a under replication queue given a priority*/
private boolean remove(Block block, int priLevel ) {
- if( priorityQueues[priLevel].remove(block) ) {
+ if( priLevel >= 0 && priLevel < LEVEL
+ && priorityQueues[priLevel].remove(block) ) {
NameNode.stateChangeLog.debug(
"BLOCK* NameSystem.UnderReplicationBlock.remove: "
+ "Removing block " + block.getBlockName()
@@ -3536,6 +3534,8 @@
NameNode.stateChangeLog.info("STATE* Network topology has "
+clusterMap.getNumOfRacks()+" racks and "
+clusterMap.getNumOfLeaves()+ " datanodes");
+ NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has "
+ +neededReplications.size()+" blocks" );
}
/**