You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/01/24 01:00:08 UTC
svn commit: r1235078 - in
/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/native/
src/main/...
Author: todd
Date: Tue Jan 24 00:00:05 2012
New Revision: 1235078
URL: http://svn.apache.org/viewvc?rev=1235078&view=rev
Log:
Merge trunk into HA branch.
- Some conflicts on BlockManager due to the recent commit of HDFS-2822 - moved logic to processMisReplicatedBlock function
- testNoExtensionIfNoBlocks(org.apache.hadoop.hdfs.TestSafeMode) is now failing due to an existing bug in the HA branch, to be fixed soon
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 24 00:00:05 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1152502-1234923
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1152502-1235069
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-1052:987665-1095512
/hadoop/hdfs/branches/HDFS-265:796829-820463
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1235078&r1=1235077&r2=1235078&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jan 24 00:00:05 2012
@@ -290,6 +290,12 @@ Release 0.23.1 - UNRELEASED
for a client on the same node as the block file. (Andrew Purtell,
Suresh Srinivas and Jitendra Nath Pandey via szetszwo)
+ HDFS-2825. Add test hook to turn off the writer preferring its local
+ DN. (todd)
+
+ HDFS-2826. Add test case for HDFS-1476 (safemode can initialize
+ replication queues before exiting) (todd)
+
BUG FIXES
HDFS-2541. For a sufficiently large value of blocks, the DN Scanner
@@ -350,6 +356,9 @@ Release 0.23.1 - UNRELEASED
HDFS-2816. Fix missing license header in httpfs findbugsExcludeFile.xml.
(hitesh via tucu)
+ HDFS-2822. processMisReplicatedBlock incorrectly identifies
+ under-construction blocks as under-replicated. (todd)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 24 00:00:05 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1159757-1234923
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1159757-1235069
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/branches/branch-0.19/hdfs/src/main/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1235078&r1=1235077&r2=1235078&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Tue Jan 24 00:00:05 2012
@@ -1877,7 +1877,8 @@ assert storedBlock.findDatanode(dn) < 0
public void processMisReplicatedBlocks() {
assert namesystem.hasWriteLock();
- long nrInvalid = 0, nrOverReplicated = 0, nrUnderReplicated = 0, nrPostponed = 0;
+ long nrInvalid = 0, nrOverReplicated = 0, nrUnderReplicated = 0, nrPostponed = 0,
+ nrUnderConstruction = 0;
neededReplications.clear();
for (BlockInfo block : blocksMap.getBlocks()) {
MisReplicationResult res = processMisReplicatedBlock(block);
@@ -1896,6 +1897,9 @@ assert storedBlock.findDatanode(dn) < 0
nrPostponed++;
postponeBlock(block);
break;
+ case UNDER_CONSTRUCTION:
+ nrUnderConstruction++;
+ break;
case OK:
break;
default:
@@ -1908,6 +1912,7 @@ assert storedBlock.findDatanode(dn) < 0
LOG.info("Number of under-replicated blocks = " + nrUnderReplicated);
LOG.info("Number of over-replicated blocks = " + nrOverReplicated +
((nrPostponed > 0) ? ( " (" + nrPostponed + " postponed)") : ""));
+ LOG.info("Number of blocks being written = " + nrUnderConstruction);
}
/**
@@ -1922,6 +1927,11 @@ assert storedBlock.findDatanode(dn) < 0
addToInvalidates(block);
return MisReplicationResult.INVALID;
}
+ if (!block.isComplete()) {
+ // Incomplete blocks are never considered mis-replicated --
+ // they'll be reached when they are completed or recovered.
+ return MisReplicationResult.UNDER_CONSTRUCTION;
+ }
// calculate current replication
short expectedReplication = fileINode.getReplication();
NumberReplicas num = countNodes(block);
@@ -2797,6 +2807,8 @@ assert storedBlock.findDatanode(dn) < 0
OVER_REPLICATED,
/** A decision can't currently be made about this block. */
POSTPONE,
+ /** The block is under construction, so should be ignored */
+ UNDER_CONSTRUCTION,
/** The block is properly replicated */
OK
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1235078&r1=1235077&r2=1235078&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Tue Jan 24 00:00:05 2012
@@ -38,6 +38,8 @@ import org.apache.hadoop.net.NetworkTopo
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
+import com.google.common.annotations.VisibleForTesting;
+
/** The class is responsible for choosing the desired number of targets
* for placing block replicas.
* The replica placement strategy is that if the writer is on a datanode,
@@ -49,6 +51,7 @@ import org.apache.hadoop.net.NodeBase;
@InterfaceAudience.Private
public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
private boolean considerLoad;
+ private boolean preferLocalNode = true;
private NetworkTopology clusterMap;
private FSClusterStats stats;
static final String enableDebugLogging = "For more information, please enable"
@@ -223,17 +226,17 @@ public class BlockPlacementPolicyDefault
if (localMachine == null)
return chooseRandom(NodeBase.ROOT, excludedNodes,
blocksize, maxNodesPerRack, results);
-
- // otherwise try local machine first
- Node oldNode = excludedNodes.put(localMachine, localMachine);
- if (oldNode == null) { // was not in the excluded list
- if (isGoodTarget(localMachine, blocksize,
- maxNodesPerRack, false, results)) {
- results.add(localMachine);
- return localMachine;
- }
- }
-
+ if (preferLocalNode) {
+ // otherwise try local machine first
+ Node oldNode = excludedNodes.put(localMachine, localMachine);
+ if (oldNode == null) { // was not in the excluded list
+ if (isGoodTarget(localMachine, blocksize,
+ maxNodesPerRack, false, results)) {
+ results.add(localMachine);
+ return localMachine;
+ }
+ }
+ }
// try a node on local rack
return chooseLocalRack(localMachine, excludedNodes,
blocksize, maxNodesPerRack, results);
@@ -568,5 +571,10 @@ public class BlockPlacementPolicyDefault
}
return cur;
}
+
+ @VisibleForTesting
+ void setPreferLocalNode(boolean prefer) {
+ this.preferLocalNode = prefer;
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1235078&r1=1235077&r2=1235078&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Jan 24 00:00:05 2012
@@ -196,6 +196,8 @@ import org.mortbay.util.ajax.JSON;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.annotations.VisibleForTesting;
+
/***************************************************
* FSNamesystem does the actual bookkeeping work for the
* DataNode.
@@ -3251,7 +3253,7 @@ public class FSNamesystem implements Nam
/** Total number of blocks. */
int blockTotal;
/** Number of safe blocks. */
- private int blockSafe;
+ int blockSafe;
/** Number of blocks needed to satisfy safe mode threshold condition */
private int blockThreshold;
/** Number of blocks needed before populating replication queues */
@@ -3259,7 +3261,7 @@ public class FSNamesystem implements Nam
/** time of the last status printout */
private long lastStatusReport = 0;
/** flag indicating whether replication queues have been initialized */
- private boolean initializedReplQueues = false;
+ boolean initializedReplQueues = false;
/** Was safemode entered automatically because available resources were low. */
private boolean resourcesLow = false;
@@ -3384,9 +3386,7 @@ public class FSNamesystem implements Nam
*/
private synchronized void initializeReplQueues() {
LOG.info("initializing replication queues");
- if (isPopulatingReplQueues()) {
- LOG.warn("Replication queues already initialized.");
- }
+ assert !isPopulatingReplQueues() : "Already initialized repl queues";
long startTimeMisReplicatedScan = now();
blockManager.processMisReplicatedBlocks();
initializedReplQueues = true;
@@ -4975,4 +4975,9 @@ public class FSNamesystem implements Nam
ReentrantReadWriteLock getFsLockForTests() {
return fsLock;
}
+
+ @VisibleForTesting
+ public SafeModeInfo getSafeModeInfoForTests() {
+ return safeMode;
+ }
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 24 00:00:05 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1152502-1234923
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1152502-1235069
/hadoop/core/branches/branch-0.19/hdfs/src/main/native:713112
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 24 00:00:05 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1159757-1234923
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1159757-1235069
/hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/datanode:713112
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
/hadoop/core/trunk/src/webapps/datanode:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 24 00:00:05 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1152502-1234923
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1152502-1235069
/hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/hdfs:713112
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 24 00:00:05 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1152502-1234923
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1152502-1235069
/hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/secondary:713112
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
/hadoop/core/trunk/src/webapps/secondary:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 24 00:00:05 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1159757-1234923
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1159757-1235069
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:987665-1095512
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java?rev=1235078&r1=1235077&r2=1235078&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java Tue Jan 24 00:00:05 2012
@@ -19,22 +19,37 @@
package org.apache.hadoop.hdfs;
import java.io.IOException;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.test.GenericTestUtils;
import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.After;
import org.junit.Test;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Lists;
+
/**
* Tests to verify safe mode correctness.
*/
public class TestSafeMode {
+ private static final Path TEST_PATH = new Path("/test");
+ private static final int BLOCK_SIZE = 1024;
Configuration conf;
MiniDFSCluster cluster;
FileSystem fs;
@@ -43,6 +58,7 @@ public class TestSafeMode {
@Before
public void startUp() throws IOException {
conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
fs = cluster.getFileSystem();
@@ -83,7 +99,7 @@ public class TestSafeMode {
// create two files with one block each.
DFSTestUtil.createFile(fs, file1, 1000, (short)1, 0);
- DFSTestUtil.createFile(fs, file2, 2000, (short)1, 0);
+ DFSTestUtil.createFile(fs, file2, 1000, (short)1, 0);
fs.close();
cluster.shutdown();
@@ -127,6 +143,106 @@ public class TestSafeMode {
String status = cluster.getNameNode().getNamesystem().getSafemode();
assertEquals("", status);
}
+
+ /**
+ * Test that the NN initializes its under-replicated blocks queue
+ * before it is ready to exit safemode (HDFS-1476)
+ */
+ @Test(timeout=45000)
+ public void testInitializeReplQueuesEarly() throws Exception {
+ // Spray the blocks around the cluster when we add DNs instead of
+ // concentrating all blocks on the first node.
+ BlockManagerTestUtil.setWritingPrefersLocalNode(
+ cluster.getNamesystem().getBlockManager(), false);
+
+ cluster.startDataNodes(conf, 2, true, StartupOption.REGULAR, null);
+ cluster.waitActive();
+ DFSTestUtil.createFile(fs, TEST_PATH, 15*BLOCK_SIZE, (short)1, 1L);
+
+
+ List<DataNodeProperties> dnprops = Lists.newLinkedList();
+ dnprops.add(cluster.stopDataNode(0));
+ dnprops.add(cluster.stopDataNode(0));
+ dnprops.add(cluster.stopDataNode(0));
+
+ cluster.getConfiguration(0).setFloat(
+ DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY, 1f/15f);
+
+ cluster.restartNameNode();
+ final NameNode nn = cluster.getNameNode();
+
+ String status = nn.getNamesystem().getSafemode();
+ assertEquals("Safe mode is ON.The reported blocks 0 needs additional " +
+ "15 blocks to reach the threshold 0.9990 of total blocks 15. " +
+ "Safe mode will be turned off automatically.", status);
+ assertFalse("Mis-replicated block queues should not be initialized " +
+ "until threshold is crossed",
+ NameNodeAdapter.safeModeInitializedReplQueues(nn));
+
+ cluster.restartDataNode(dnprops.remove(0));
+
+ // Wait for the block report from the restarted DN to come in.
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return NameNodeAdapter.getSafeModeSafeBlocks(nn) > 0;
+ }
+ }, 10, 10000);
+ // SafeMode is fine-grain synchronized, so the processMisReplicatedBlocks
+ // call is still going on at this point - wait until it's done by grabbing
+ // the lock.
+ nn.getNamesystem().writeLock();
+ nn.getNamesystem().writeUnlock();
+ int safe = NameNodeAdapter.getSafeModeSafeBlocks(nn);
+ assertTrue("Expected first block report to make some but not all blocks " +
+ "safe. Got: " + safe, safe >= 1 && safe < 15);
+ BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
+
+ assertTrue(NameNodeAdapter.safeModeInitializedReplQueues(nn));
+ assertEquals(15 - safe, nn.getNamesystem().getUnderReplicatedBlocks());
+
+ cluster.restartDataNodes();
+ }
+
+ /**
+ * Test that, when under-replicated blocks are processed at the end of
+ * safe-mode, blocks currently under construction are not considered
+ * under-construction or missing. Regression test for HDFS-2822.
+ */
+ @Test
+ public void testRbwBlocksNotConsideredUnderReplicated() throws IOException {
+ List<FSDataOutputStream> stms = Lists.newArrayList();
+ try {
+ // Create some junk blocks so that the NN doesn't just immediately
+ // exit safemode on restart.
+ DFSTestUtil.createFile(fs, new Path("/junk-blocks"),
+ BLOCK_SIZE*4, (short)1, 1L);
+ // Create several files which are left open. It's important to
+ // create several here, because otherwise the first iteration of the
+ // replication monitor will pull them off the replication queue and
+ // hide this bug from the test!
+ for (int i = 0; i < 10; i++) {
+ FSDataOutputStream stm = fs.create(
+ new Path("/append-" + i), true, BLOCK_SIZE, (short) 1, BLOCK_SIZE);
+ stms.add(stm);
+ stm.write(1);
+ stm.hflush();
+ }
+
+ cluster.restartNameNode();
+ FSNamesystem ns = cluster.getNameNode(0).getNamesystem();
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ assertEquals(0, ns.getPendingReplicationBlocks());
+ assertEquals(0, ns.getCorruptReplicaBlocks());
+ assertEquals(0, ns.getMissingBlocksCount());
+
+ } finally {
+ for (FSDataOutputStream stm : stms) {
+ IOUtils.closeStream(stm);
+ }
+ cluster.shutdown();
+ }
+ }
public interface FSRun {
public abstract void run(FileSystem fs) throws IOException;
@@ -241,4 +357,4 @@ public class TestSafeMode {
assertEquals("", cluster.getNamesystem().getSafemode());
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java?rev=1235078&r1=1235077&r2=1235078&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java Tue Jan 24 00:00:05 2012
@@ -30,6 +30,8 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.util.Daemon;
import org.junit.Assert;
+import com.google.common.base.Preconditions;
+
public class BlockManagerTestUtil {
public static void setNodeReplicationLimit(final BlockManager blockManager,
final int limit) {
@@ -177,4 +179,17 @@ public class BlockManagerTestUtil {
namesystem.writeUnlock();
}
}
+
+ /**
+ * Change whether the block placement policy will prefer the writer's
+ * local Datanode or not.
+ * @param prefer
+ */
+ public static void setWritingPrefersLocalNode(
+ BlockManager bm, boolean prefer) {
+ BlockPlacementPolicy bpp = bm.getBlockPlacementPolicy();
+ Preconditions.checkState(bpp instanceof BlockPlacementPolicyDefault,
+ "Must use default policy, got %s", bpp.getClass());
+ ((BlockPlacementPolicyDefault)bpp).setPreferLocalNode(prefer);
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1235078&r1=1235077&r2=1235078&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Tue Jan 24 00:00:05 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.SafeModeInfo;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.ipc.Server;
@@ -179,4 +180,28 @@ public class NameNodeAdapter {
return null;
}
}
+
+ /**
+ * @return the number of blocks marked safe by safemode, or -1
+ * if safemode is not running.
+ */
+ public static int getSafeModeSafeBlocks(NameNode nn) {
+ SafeModeInfo smi = nn.getNamesystem().getSafeModeInfoForTests();
+ if (smi == null) {
+ return -1;
+ }
+ return smi.blockSafe;
+ }
+
+ /**
+ * @return true if safemode is not running, or if safemode has already
+ * initialized the replication queues
+ */
+ public static boolean safeModeInitializedReplQueues(NameNode nn) {
+ SafeModeInfo smi = nn.getNamesystem().getSafeModeInfoForTests();
+ if (smi == null) {
+ return true;
+ }
+ return smi.initializedReplQueues;
+ }
}