You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2013/04/29 04:07:58 UTC
svn commit: r1476877 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt
src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
Author: suresh
Date: Mon Apr 29 02:07:58 2013
New Revision: 1476877
URL: http://svn.apache.org/r1476877
Log:
HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows. Contributed by Arpit Agarwal.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1476877&r1=1476876&r2=1476877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Apr 29 02:07:58 2013
@@ -326,6 +326,9 @@ Trunk (Unreleased)
HDFS-4705. Address HDFS test failures on Windows because of invalid
dfs.namenode.name.dir. (Ivan Mitic via suresh)
+ HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows.
+ (Arpit Agarwal via suresh)
+
BREAKDOWN OF HDFS-347 SUBTASKS AND RELATED JIRAS
HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes.
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java?rev=1476877&r1=1476876&r2=1476877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java Mon Apr 29 02:07:58 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.test.MockitoUtil;
+import org.apache.hadoop.util.Shell;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
@@ -73,6 +74,17 @@ public class TestDFSHAAdmin {
private static String HOST_A = "1.2.3.1";
private static String HOST_B = "1.2.3.2";
+ // Fencer shell commands that always return true and false respectively
+ // on Unix.
+ private static String FENCER_TRUE_COMMAND_UNIX = "shell(true)";
+ private static String FENCER_FALSE_COMMAND_UNIX = "shell(false)";
+
+ // Fencer shell commands that always return true and false respectively
+ // on Windows. Lacking POSIX 'true' and 'false' commands we use the DOS
+ // commands 'rem' and 'help.exe'.
+ private static String FENCER_TRUE_COMMAND_WINDOWS = "shell(rem)";
+ private static String FENCER_FALSE_COMMAND_WINDOWS = "shell(help.exe /? >NUL)";
+
private HdfsConfiguration getHAConf() {
HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID);
@@ -89,6 +101,16 @@ public class TestDFSHAAdmin {
return conf;
}
+ public static String getFencerTrueCommand() {
+ return Shell.WINDOWS ?
+ FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX;
+ }
+
+ public static String getFencerFalseCommand() {
+ return Shell.WINDOWS ?
+ FENCER_FALSE_COMMAND_WINDOWS : FENCER_FALSE_COMMAND_UNIX;
+ }
+
@Before
public void setup() throws IOException {
mockProtocol = MockitoUtil.mockProtocol(HAServiceProtocol.class);
@@ -173,7 +195,7 @@ public class TestDFSHAAdmin {
// Turn on auto-HA in the config
HdfsConfiguration conf = getHAConf();
conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
// Should fail without the forcemanual flag
@@ -250,7 +272,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithFencerConfigured() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2"));
}
@@ -259,7 +281,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithFencerAndNameservice() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-ns", "ns1", "-failover", "nn1", "nn2"));
}
@@ -268,7 +290,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithFencerConfiguredAndForce() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
}
@@ -277,7 +299,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithForceActive() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forceactive"));
}
@@ -286,7 +308,7 @@ public class TestDFSHAAdmin {
public void testFailoverWithInvalidFenceArg() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(-1, runTool("-failover", "nn1", "nn2", "notforcefence"));
}
@@ -312,7 +334,7 @@ public class TestDFSHAAdmin {
// Turn on auto-HA in the config
HdfsConfiguration conf = getHAConf();
conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2"));
@@ -323,7 +345,7 @@ public class TestDFSHAAdmin {
public void testForceFenceOptionListedBeforeArgs() throws Exception {
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
}
@@ -359,23 +381,23 @@ public class TestDFSHAAdmin {
HdfsConfiguration conf = getHAConf();
// Set the default fencer to succeed
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
// Set the NN-specific fencer to fail. Should fail to fence.
- conf.set(nnSpecificKey, "shell(false)");
+ conf.set(nnSpecificKey, getFencerFalseCommand());
tool.setConf(conf);
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
conf.unset(nnSpecificKey);
// Set an NS-specific fencer to fail. Should fail.
- conf.set(nsSpecificKey, "shell(false)");
+ conf.set(nsSpecificKey, getFencerFalseCommand());
tool.setConf(conf);
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
// Set the NS-specific fencer to succeed. Should succeed
- conf.set(nsSpecificKey, "shell(true)");
+ conf.set(nsSpecificKey, getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java?rev=1476877&r1=1476876&r2=1476877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java Mon Apr 29 02:07:58 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.util.Shell;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Before;
@@ -114,7 +115,8 @@ public class TestDFSHAAdminMiniCluster {
@Test
public void testTryFailoverToSafeMode() throws Exception {
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
+ TestDFSHAAdmin.getFencerTrueCommand());
tool.setConf(conf);
NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false);
@@ -136,10 +138,17 @@ public class TestDFSHAAdminMiniCluster {
// tmp file, so we can verify that the args were substituted right
File tmpFile = File.createTempFile("testFencer", ".txt");
tmpFile.deleteOnExit();
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
- "shell(echo -n $target_nameserviceid.$target_namenodeid " +
- "$target_port $dfs_ha_namenode_id > " +
- tmpFile.getAbsolutePath() + ")");
+ if (Shell.WINDOWS) {
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
+ "shell(echo %target_nameserviceid%.%target_namenodeid% " +
+ "%target_port% %dfs_ha_namenode_id% > " +
+ tmpFile.getAbsolutePath() + ")");
+ } else {
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
+ "shell(echo -n $target_nameserviceid.$target_namenodeid " +
+ "$target_port $dfs_ha_namenode_id > " +
+ tmpFile.getAbsolutePath() + ")");
+ }
// Test failover with fencer
tool.setConf(conf);
@@ -156,9 +165,11 @@ public class TestDFSHAAdminMiniCluster {
assertEquals(0, runTool("-failover", "nn1", "nn2", "--forcefence"));
// The fence script should run with the configuration from the target
- // node, rather than the configuration from the fencing node
- assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1",
- Files.toString(tmpFile, Charsets.UTF_8));
+ // node, rather than the configuration from the fencing node. Strip
+ // out any trailing spaces and CR/LFs which may be present on Windows.
+ String fenceCommandOutput =Files.toString(tmpFile, Charsets.UTF_8).
+ replaceAll(" *[\r\n]+", "");
+ assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1", fenceCommandOutput);
tmpFile.delete();
// Test failover with forceactive option
@@ -181,7 +192,8 @@ public class TestDFSHAAdminMiniCluster {
assertFalse(tmpFile.exists());
// Test failover with force fence listed before the other arguments
- conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,
+ TestDFSHAAdmin.getFencerTrueCommand());
tool.setConf(conf);
assertEquals(0, runTool("-failover", "--forcefence", "nn1", "nn2"));
}