You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sh...@apache.org on 2009/09/21 01:24:28 UTC
svn commit: r817122 - in /hadoop/hdfs/branches/HDFS-265: ./
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/
Author: shv
Date: Sun Sep 20 23:24:28 2009
New Revision: 817122
URL: http://svn.apache.org/viewvc?rev=817122&view=rev
Log:
HDFS-636. SafeMode counts complete blocks only. Contributed by Konstantin Shvachko.
Modified:
hadoop/hdfs/branches/HDFS-265/CHANGES.txt
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java
Modified: hadoop/hdfs/branches/HDFS-265/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/CHANGES.txt?rev=817122&r1=817121&r2=817122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-265/CHANGES.txt Sun Sep 20 23:24:28 2009
@@ -8,6 +8,8 @@
HDFS-576. Block report includes under-construction replicas. (shv)
+ HDFS-636. SafeMode counts complete blocks only. (shv)
+
NEW FEATURES
HDFS-536. Support hflush at DFSClient. (hairong)
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=817122&r1=817121&r2=817122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Sun Sep 20 23:24:28 2009
@@ -1102,15 +1102,18 @@
int numCurrentReplica = numLiveReplicas
+ pendingReplications.getNumReplicas(storedBlock);
+ if(storedBlock.getBlockUCState() == BlockUCState.COMMITTED &&
+ numLiveReplicas >= minReplication)
+ storedBlock = completeBlock(fileINode, storedBlock);
+
// check whether safe replication is reached for the block
- namesystem.incrementSafeBlockCount(numCurrentReplica);
+ // only complete blocks are counted towards that
+ if(!storedBlock.isUnderConstruction())
+ namesystem.incrementSafeBlockCount(numCurrentReplica);
// if file is under construction, then check whether the block
// can be completed
if (fileINode.isUnderConstruction()) {
- if(storedBlock.getBlockUCState() == BlockUCState.COMMITTED &&
- numLiveReplicas >= minReplication)
- storedBlock = completeBlock(fileINode, storedBlock);
return storedBlock;
}
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=817122&r1=817121&r2=817122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sun Sep 20 23:24:28 2009
@@ -3371,7 +3371,7 @@
void setBlockTotal() {
if (safeMode == null)
return;
- safeMode.setBlockTotal((int)getBlocksTotal());
+ safeMode.setBlockTotal((int)getCompleteBlocksTotal());
}
/**
@@ -3382,6 +3382,33 @@
}
/**
+ * Get the total number of COMPLETE blocks in the system.
+ * For safe mode only complete blocks are counted.
+ */
+ long getCompleteBlocksTotal() {
+ // Calculate number of blocks under construction
+ long numUCBlocks = 0;
+ for (Lease lease : leaseManager.getSortedLeases()) {
+ for(String path : lease.getPaths()) {
+ INode node = dir.getFileINode(path);
+ assert node != null : "Found a lease for nonexisting file.";
+ assert node.isUnderConstruction() :
+ "Found a lease for file that is not under construction.";
+ INodeFileUnderConstruction cons = (INodeFileUnderConstruction) node;
+ BlockInfo[] blocks = cons.getBlocks();
+ if(blocks == null)
+ continue;
+ for(BlockInfo b : blocks) {
+ if(b.isUnderConstruction())
+ numUCBlocks++;
+ }
+ }
+ }
+ LOG.info("Number of blocks under construction: " + numUCBlocks);
+ return getBlocksTotal() - numUCBlocks;
+ }
+
+ /**
* Enter safe mode manually.
* @throws IOException
*/
Modified: hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java?rev=817122&r1=817121&r2=817122&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/TestRenameWhileOpen.java Sun Sep 20 23:24:28 2009
@@ -30,8 +30,6 @@
import org.apache.log4j.Level;
public class TestRenameWhileOpen extends junit.framework.TestCase {
- private static final long LEASE_PERIOD = 500L;
-
{
((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
@@ -54,7 +52,7 @@
conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME);
conf.setInt("heartbeat.recheck.interval", 1000);
conf.setInt("dfs.heartbeat.interval", 1);
- conf.setFloat("dfs.safemode.threshold.pct", 0.5f);
+ conf.setInt("dfs.safemode.threshold.pct", 1);
conf.setBoolean("dfs.support.append", true);
// create cluster
@@ -106,7 +104,7 @@
try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {}
cluster = new MiniDFSCluster(nnport, conf, 1, false, true,
null, null, null);
- waitLeaseRecovery(cluster);
+ cluster.waitActive();
// restart cluster yet again. This triggers the code to read in
// persistent leases from fsimage.
@@ -138,7 +136,7 @@
conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME);
conf.setInt("heartbeat.recheck.interval", 1000);
conf.setInt("dfs.heartbeat.interval", 1);
- conf.setFloat("dfs.safemode.threshold.pct", 0.5f);
+ conf.setInt("dfs.safemode.threshold.pct", 1);
conf.setBoolean("dfs.support.append", true);
System.out.println("Test 2************************************");
@@ -178,7 +176,7 @@
try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {}
cluster = new MiniDFSCluster(nnport, conf, 1, false, true,
null, null, null);
- waitLeaseRecovery(cluster);
+ cluster.waitActive();
// restart cluster yet again. This triggers the code to read in
// persistent leases from fsimage.
@@ -211,7 +209,7 @@
conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME);
conf.setInt("heartbeat.recheck.interval", 1000);
conf.setInt("dfs.heartbeat.interval", 1);
- conf.setFloat("dfs.safemode.threshold.pct", 0.5f);
+ conf.setInt("dfs.safemode.threshold.pct", 1);
conf.setBoolean("dfs.support.append", true);
System.out.println("Test 3************************************");
@@ -243,7 +241,7 @@
try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {}
cluster = new MiniDFSCluster(nnport, conf, 1, false, true,
null, null, null);
- waitLeaseRecovery(cluster);
+ cluster.waitActive();
// restart cluster yet again. This triggers the code to read in
// persistent leases from fsimage.
@@ -274,7 +272,7 @@
conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME);
conf.setInt("heartbeat.recheck.interval", 1000);
conf.setInt("dfs.heartbeat.interval", 1);
- conf.setFloat("dfs.safemode.threshold.pct", 0.5f);
+ conf.setInt("dfs.safemode.threshold.pct", 1);
conf.setBoolean("dfs.support.append", true);
System.out.println("Test 4************************************");
@@ -305,7 +303,7 @@
try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {}
cluster = new MiniDFSCluster(nnport, conf, 1, false, true,
null, null, null);
- waitLeaseRecovery(cluster);
+ cluster.waitActive();
// restart cluster yet again. This triggers the code to read in
// persistent leases from fsimage.
@@ -325,13 +323,4 @@
cluster.shutdown();
}
}
-
- void waitLeaseRecovery(MiniDFSCluster cluster) {
- cluster.setLeasePeriod(LEASE_PERIOD, LEASE_PERIOD);
- // wait for the lease to expire
- try {
- Thread.sleep(5 * LEASE_PERIOD);
- } catch (InterruptedException e) {
- }
- }
}