You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/06/06 02:18:04 UTC
svn commit: r1346682 [7/9] - in
/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project: ./ hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs/ hadoop-hdfs/dev-support/ hadoop-hdfs/src/contrib/bkjournal/
ha...
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Wed Jun 6 00:17:38 2012
@@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
@@ -517,14 +518,15 @@ public class TestSaveNamespace {
try {
doAnEdit(fsn, 1);
-
+ final Canceler canceler = new Canceler();
+
// Save namespace
fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
try {
Future<Void> saverFuture = pool.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
- image.saveNamespace(finalFsn);
+ image.saveNamespace(finalFsn, canceler);
return null;
}
});
@@ -534,7 +536,7 @@ public class TestSaveNamespace {
// then cancel the saveNamespace
Future<Void> cancelFuture = pool.submit(new Callable<Void>() {
public Void call() throws Exception {
- image.cancelSaveNamespace("cancelled");
+ canceler.cancel("cancelled");
return null;
}
});
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java Wed Jun 6 00:17:38 2012
@@ -92,7 +92,7 @@ public class TestValidateConfigurationSe
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
"127.0.0.1:0");
- conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "ns1");
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns1");
// Set a nameservice-specific configuration for name dir
File dir = new File(MiniDFSCluster.getBaseDirectory(),
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java Wed Jun 6 00:17:38 2012
@@ -185,7 +185,7 @@ public abstract class HATestUtil {
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
logicalName, nameNodeId2), address2);
- conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, logicalName);
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName);
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, logicalName),
nameNodeId1 + "," + nameNodeId2);
conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + logicalName,
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java Wed Jun 6 00:17:38 2012
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.net.URI;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.junit.After;
@@ -43,6 +45,7 @@ import org.junit.Test;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import static org.junit.Assert.*;
@@ -177,7 +180,7 @@ public class TestBootstrapStandby {
logs.stopCapturing();
}
GenericTestUtils.assertMatches(logs.getOutput(),
- "FATAL.*Unable to read transaction ids 1-4 from the configured shared");
+ "FATAL.*Unable to read transaction ids 1-3 from the configured shared");
}
@Test
@@ -195,30 +198,29 @@ public class TestBootstrapStandby {
assertEquals(0, rc);
}
+ /**
+ * Test that, even if the other node is not active, we are able
+ * to bootstrap standby from it.
+ */
@Test(timeout=30000)
public void testOtherNodeNotActive() throws Exception {
cluster.transitionToStandby(0);
int rc = BootstrapStandby.run(
- new String[]{"-nonInteractive"},
- cluster.getConfiguration(1));
- assertEquals(BootstrapStandby.ERR_CODE_OTHER_NN_NOT_ACTIVE, rc);
-
- // Answer "yes" to the prompt about transition to active
- System.setIn(new ByteArrayInputStream("yes\n".getBytes()));
- rc = BootstrapStandby.run(
new String[]{"-force"},
cluster.getConfiguration(1));
assertEquals(0, rc);
-
- assertFalse(nn0.getNamesystem().isInStandbyState());
}
-
+
private void assertNNFilesMatch() throws Exception {
List<File> curDirs = Lists.newArrayList();
curDirs.addAll(FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0));
curDirs.addAll(FSImageTestUtil.getNameNodeCurrentDirs(cluster, 1));
+
+ // Ignore seen_txid file, since the newly bootstrapped standby
+ // will have a higher seen_txid than the one it bootstrapped from.
+ Set<String> ignoredFiles = ImmutableSet.of("seen_txid");
FSImageTestUtil.assertParallelFilesAreIdentical(curDirs,
- Collections.<String>emptySet());
+ ignoredFiles);
}
private void removeStandbyNameDirs() {
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java Wed Jun 6 00:17:38 2012
@@ -71,7 +71,7 @@ public class TestEditLogsDuringFailover
// Set the first NN to active, make sure it creates edits
// in its own dirs and the shared dir. The standby
// should still have no edits!
- cluster.getNameNode(0).getRpcServer().transitionToActive();
+ cluster.transitionToActive(0);
assertEditFiles(cluster.getNameDirs(0),
NNStorage.getInProgressEditsFileName(1));
@@ -107,7 +107,7 @@ public class TestEditLogsDuringFailover
// If we restart NN0, it'll come back as standby, and we can
// transition NN1 to active and make sure it reads edits correctly at this point.
cluster.restartNameNode(0);
- cluster.getNameNode(1).getRpcServer().transitionToActive();
+ cluster.transitionToActive(1);
// NN1 should have both the edits that came before its restart, and the edits that
// came after its restart.
@@ -134,7 +134,7 @@ public class TestEditLogsDuringFailover
NNStorage.getInProgressEditsFileName(1));
// Transition one of the NNs to active
- cluster.getNameNode(0).getRpcServer().transitionToActive();
+ cluster.transitionToActive(0);
// In the transition to active, it should have read the log -- and
// hence see one of the dirs we made in the fake log.
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java Wed Jun 6 00:17:38 2012
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
+import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.junit.After;
@@ -256,16 +258,21 @@ public class TestFailureToReadEdits {
// Shutdown the active NN.
cluster.shutdownNameNode(0);
+ Runtime mockRuntime = mock(Runtime.class);
+ cluster.getNameNode(1).setRuntimeForTesting(mockRuntime);
+ verify(mockRuntime, times(0)).exit(anyInt());
try {
// Transition the standby to active.
cluster.transitionToActive(1);
fail("Standby transitioned to active, but should not have been able to");
} catch (ServiceFailedException sfe) {
- LOG.info("got expected exception: " + sfe.toString(), sfe);
+ Throwable sfeCause = sfe.getCause();
+ LOG.info("got expected exception: " + sfeCause.toString(), sfeCause);
assertTrue("Standby failed to catch up for some reason other than "
- + "failure to read logs", sfe.toString().contains(
+ + "failure to read logs", sfeCause.getCause().toString().contains(
EditLogInputException.class.getName()));
}
+ verify(mockRuntime, times(1)).exit(anyInt());
}
private LimitedEditLogAnswer causeFailureOnEditLogRead() throws IOException {
@@ -273,7 +280,7 @@ public class TestFailureToReadEdits {
.getEditLog());
LimitedEditLogAnswer answer = new LimitedEditLogAnswer();
doAnswer(answer).when(spyEditLog).selectInputStreams(
- anyLong(), anyLong(), anyBoolean());
+ anyLong(), anyLong(), (MetaRecoveryContext)anyObject(), anyBoolean());
nn1.getNamesystem().getEditLogTailer().setEditLog(spyEditLog);
return answer;
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java Wed Jun 6 00:17:38 2012
@@ -57,7 +57,7 @@ public class TestHAConfiguration {
private Configuration getHAConf(String nsId, String host1, String host2) {
Configuration conf = new Configuration();
- conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, nsId);
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, nsId);
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, nsId),
"nn1,nn2");
@@ -72,10 +72,10 @@ public class TestHAConfiguration {
}
@Test
- public void testGetOtherNNHttpAddress() {
+ public void testGetOtherNNHttpAddress() throws IOException {
// Use non-local addresses to avoid host address matching
Configuration conf = getHAConf("ns1", "1.2.3.1", "1.2.3.2");
- conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID, "ns1");
+ conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, "ns1");
// This is done by the NN before the StandbyCheckpointer is created
NameNode.initializeGenericKeys(conf, "ns1", "nn1");
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java Wed Jun 6 00:17:38 2012
@@ -34,6 +34,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
+import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HAUtil;
@@ -129,7 +131,8 @@ public class TestHASafeMode {
DFSTestUtil
.createFile(fs, new Path("/test"), 3 * BLOCK_SIZE, (short) 3, 1L);
restartActive();
- nn0.getRpcServer().transitionToActive();
+ nn0.getRpcServer().transitionToActive(
+ new StateChangeRequestInfo(RequestSource.REQUEST_BY_USER));
FSNamesystem namesystem = nn0.getNamesystem();
String status = namesystem.getSafemode();
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java Wed Jun 6 00:17:38 2012
@@ -37,6 +37,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
+import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -71,6 +73,8 @@ public class TestHAStateTransitions {
private static final String TEST_FILE_STR = TEST_FILE_PATH.toUri().getPath();
private static final String TEST_FILE_DATA =
"Hello state transitioning world";
+ private static final StateChangeRequestInfo REQ_INFO = new StateChangeRequestInfo(
+ RequestSource.REQUEST_BY_USER_FORCED);
static {
((Log4JLogger)EditLogTailer.LOG).getLogger().setLevel(Level.ALL);
@@ -481,19 +485,19 @@ public class TestHAStateTransitions {
assertFalse(isDTRunning(nn));
banner("Transition 1->3. Should not start secret manager.");
- nn.getRpcServer().transitionToActive();
+ nn.getRpcServer().transitionToActive(REQ_INFO);
assertFalse(nn.isStandbyState());
assertTrue(nn.isInSafeMode());
assertFalse(isDTRunning(nn));
banner("Transition 3->1. Should not start secret manager.");
- nn.getRpcServer().transitionToStandby();
+ nn.getRpcServer().transitionToStandby(REQ_INFO);
assertTrue(nn.isStandbyState());
assertTrue(nn.isInSafeMode());
assertFalse(isDTRunning(nn));
banner("Transition 1->3->4. Should start secret manager.");
- nn.getRpcServer().transitionToActive();
+ nn.getRpcServer().transitionToActive(REQ_INFO);
NameNodeAdapter.leaveSafeMode(nn, false);
assertFalse(nn.isStandbyState());
assertFalse(nn.isInSafeMode());
@@ -514,13 +518,13 @@ public class TestHAStateTransitions {
for (int i = 0; i < 20; i++) {
// Loop the last check to suss out races.
banner("Transition 4->2. Should stop secret manager.");
- nn.getRpcServer().transitionToStandby();
+ nn.getRpcServer().transitionToStandby(REQ_INFO);
assertTrue(nn.isStandbyState());
assertFalse(nn.isInSafeMode());
assertFalse(isDTRunning(nn));
banner("Transition 2->4. Should start secret manager");
- nn.getRpcServer().transitionToActive();
+ nn.getRpcServer().transitionToActive(REQ_INFO);
assertFalse(nn.isStandbyState());
assertFalse(nn.isInSafeMode());
assertTrue(isDTRunning(nn));
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java Wed Jun 6 00:17:38 2012
@@ -27,6 +27,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
+import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -111,7 +113,8 @@ public class TestInitializeSharedEdits {
cluster.restartNameNode(1, true);
// Make sure HA is working.
- cluster.getNameNode(0).getRpcServer().transitionToActive();
+ cluster.getNameNode(0).getRpcServer().transitionToActive(
+ new StateChangeRequestInfo(RequestSource.REQUEST_BY_USER));
FileSystem fs = null;
try {
Path newPath = new Path(TEST_PATH, pathSuffix);
@@ -160,7 +163,7 @@ public class TestInitializeSharedEdits {
@Test
public void testInitializeSharedEditsConfiguresGenericConfKeys() {
Configuration conf = new Configuration();
- conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "ns1");
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns1");
conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX,
"ns1"), "nn1,nn2");
conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY,
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestNNHealthCheck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestNNHealthCheck.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestNNHealthCheck.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestNNHealthCheck.java Wed Jun 6 00:17:38 2012
@@ -22,6 +22,8 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
+import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java Wed Jun 6 00:17:38 2012
@@ -85,12 +85,52 @@ public class TestPipelinesFailover {
private static final int STRESS_NUM_THREADS = 25;
private static final int STRESS_RUNTIME = 40000;
+
+ enum TestScenario {
+ GRACEFUL_FAILOVER {
+ void run(MiniDFSCluster cluster) throws IOException {
+ cluster.transitionToStandby(0);
+ cluster.transitionToActive(1);
+ }
+ },
+ ORIGINAL_ACTIVE_CRASHED {
+ void run(MiniDFSCluster cluster) throws IOException {
+ cluster.restartNameNode(0);
+ cluster.transitionToActive(1);
+ }
+ };
+
+ abstract void run(MiniDFSCluster cluster) throws IOException;
+ }
+
+ enum MethodToTestIdempotence {
+ ALLOCATE_BLOCK,
+ COMPLETE_FILE;
+ }
/**
* Tests continuing a write pipeline over a failover.
*/
@Test(timeout=30000)
- public void testWriteOverFailover() throws Exception {
+ public void testWriteOverGracefulFailover() throws Exception {
+ doWriteOverFailoverTest(TestScenario.GRACEFUL_FAILOVER,
+ MethodToTestIdempotence.ALLOCATE_BLOCK);
+ }
+
+ @Test(timeout=30000)
+ public void testAllocateBlockAfterCrashFailover() throws Exception {
+ doWriteOverFailoverTest(TestScenario.ORIGINAL_ACTIVE_CRASHED,
+ MethodToTestIdempotence.ALLOCATE_BLOCK);
+ }
+
+ @Test(timeout=30000)
+ public void testCompleteFileAfterCrashFailover() throws Exception {
+ doWriteOverFailoverTest(TestScenario.ORIGINAL_ACTIVE_CRASHED,
+ MethodToTestIdempotence.COMPLETE_FILE);
+ }
+
+ private void doWriteOverFailoverTest(TestScenario scenario,
+ MethodToTestIdempotence methodToTest) throws Exception {
Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
// Don't check replication periodically.
@@ -102,6 +142,8 @@ public class TestPipelinesFailover {
.numDataNodes(3)
.build();
try {
+ int sizeWritten = 0;
+
cluster.waitActive();
cluster.transitionToActive(0);
Thread.sleep(500);
@@ -112,28 +154,39 @@ public class TestPipelinesFailover {
// write a block and a half
AppendTestUtil.write(stm, 0, BLOCK_AND_A_HALF);
+ sizeWritten += BLOCK_AND_A_HALF;
// Make sure all of the blocks are written out before failover.
stm.hflush();
LOG.info("Failing over to NN 1");
- cluster.transitionToStandby(0);
- cluster.transitionToActive(1);
+ scenario.run(cluster);
- assertTrue(fs.exists(TEST_PATH));
+ // NOTE: explicitly do *not* make any further metadata calls
+ // to the NN here. The next IPC call should be to allocate the next
+ // block. Any other call would notice the failover and not test
+ // idempotence of the operation (HDFS-3031)
+
FSNamesystem ns1 = cluster.getNameNode(1).getNamesystem();
BlockManagerTestUtil.updateState(ns1.getBlockManager());
assertEquals(0, ns1.getPendingReplicationBlocks());
assertEquals(0, ns1.getCorruptReplicaBlocks());
assertEquals(0, ns1.getMissingBlocksCount());
- // write another block and a half
- AppendTestUtil.write(stm, BLOCK_AND_A_HALF, BLOCK_AND_A_HALF);
-
+ // If we're testing allocateBlock()'s idempotence, write another
+ // block and a half, so we have to allocate a new block.
+ // Otherise, don't write anything, so our next RPC will be
+ // completeFile() if we're testing idempotence of that operation.
+ if (methodToTest == MethodToTestIdempotence.ALLOCATE_BLOCK) {
+ // write another block and a half
+ AppendTestUtil.write(stm, sizeWritten, BLOCK_AND_A_HALF);
+ sizeWritten += BLOCK_AND_A_HALF;
+ }
+
stm.close();
stm = null;
- AppendTestUtil.check(fs, TEST_PATH, BLOCK_SIZE * 3);
+ AppendTestUtil.check(fs, TEST_PATH, sizeWritten);
} finally {
IOUtils.closeStream(stm);
cluster.shutdown();
@@ -146,7 +199,18 @@ public class TestPipelinesFailover {
* even when the pipeline was constructed on a different NN.
*/
@Test(timeout=30000)
- public void testWriteOverFailoverWithDnFail() throws Exception {
+ public void testWriteOverGracefulFailoverWithDnFail() throws Exception {
+ doTestWriteOverFailoverWithDnFail(TestScenario.GRACEFUL_FAILOVER);
+ }
+
+ @Test(timeout=30000)
+ public void testWriteOverCrashFailoverWithDnFail() throws Exception {
+ doTestWriteOverFailoverWithDnFail(TestScenario.ORIGINAL_ACTIVE_CRASHED);
+ }
+
+
+ private void doTestWriteOverFailoverWithDnFail(TestScenario scenario)
+ throws Exception {
Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
@@ -171,8 +235,7 @@ public class TestPipelinesFailover {
stm.hflush();
LOG.info("Failing over to NN 1");
- cluster.transitionToStandby(0);
- cluster.transitionToActive(1);
+ scenario.run(cluster);
assertTrue(fs.exists(TEST_PATH));
@@ -183,8 +246,8 @@ public class TestPipelinesFailover {
stm.hflush();
LOG.info("Failing back to NN 0");
- cluster.transitionToStandby(0);
- cluster.transitionToActive(1);
+ cluster.transitionToStandby(1);
+ cluster.transitionToActive(0);
cluster.stopDataNode(1);
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java Wed Jun 6 00:17:38 2012
@@ -21,6 +21,7 @@ import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
+import java.io.OutputStream;
import java.net.URI;
import java.util.List;
@@ -36,6 +37,11 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.hdfs.util.Canceler;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -48,16 +54,22 @@ import com.google.common.collect.Lists;
public class TestStandbyCheckpoints {
private static final int NUM_DIRS_IN_LOG = 200000;
- private MiniDFSCluster cluster;
- private NameNode nn0, nn1;
- private FileSystem fs;
+ protected MiniDFSCluster cluster;
+ protected NameNode nn0, nn1;
+ protected FileSystem fs;
+ @SuppressWarnings("rawtypes")
@Before
public void setupCluster() throws Exception {
Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 5);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true);
+ conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY,
+ SlowCodec.class.getCanonicalName());
+ CompressionCodecFactory.setCodecClasses(conf,
+ ImmutableList.<Class>of(SlowCodec.class));
MiniDFSNNTopology topology = new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf("ns1")
@@ -159,14 +171,15 @@ public class TestStandbyCheckpoints {
// We should make exactly one checkpoint at this new txid.
Mockito.verify(spyImage1, Mockito.times(1))
- .saveNamespace((FSNamesystem) Mockito.anyObject());
+ .saveNamespace((FSNamesystem) Mockito.anyObject(),
+ (Canceler)Mockito.anyObject());
}
/**
* Test cancellation of ongoing checkpoints when failover happens
* mid-checkpoint.
*/
- @Test
+ @Test(timeout=120000)
public void testCheckpointCancellation() throws Exception {
cluster.transitionToStandby(0);
@@ -191,16 +204,18 @@ public class TestStandbyCheckpoints {
cluster.transitionToActive(0);
- for (int i = 0; i < 10; i++) {
+ boolean canceledOne = false;
+ for (int i = 0; i < 10 && !canceledOne; i++) {
doEdits(i*10, i*10 + 10);
cluster.transitionToStandby(0);
cluster.transitionToActive(1);
cluster.transitionToStandby(1);
cluster.transitionToActive(0);
+ canceledOne = StandbyCheckpointer.getCanceledCount() > 0;
}
- assertTrue(StandbyCheckpointer.getCanceledCount() > 0);
+ assertTrue(canceledOne);
}
private void doEdits(int start, int stop) throws IOException {
@@ -209,5 +224,22 @@ public class TestStandbyCheckpoints {
fs.mkdirs(p);
}
}
+
+ /**
+ * A codec which just slows down the saving of the image significantly
+ * by sleeping a few milliseconds on every write. This makes it easy to
+ * catch the standby in the middle of saving a checkpoint.
+ */
+ public static class SlowCodec extends GzipCodec {
+ @Override
+ public CompressionOutputStream createOutputStream(OutputStream out)
+ throws IOException {
+ CompressionOutputStream ret = super.createOutputStream(out);
+ CompressionOutputStream spy = Mockito.spy(ret);
+ Mockito.doAnswer(new GenericTestUtils.SleepAnswer(2))
+ .when(spy).write(Mockito.<byte[]>any(), Mockito.anyInt(), Mockito.anyInt());
+ return spy;
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java Wed Jun 6 00:17:38 2012
@@ -42,7 +42,6 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@@ -70,7 +69,7 @@ public class TestStandbyIsHot {
((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
}
- @Test
+ @Test(timeout=60000)
public void testStandbyIsHot() throws Exception {
Configuration conf = new Configuration();
// We read from the standby to watch block locations
@@ -111,6 +110,8 @@ public class TestStandbyIsHot {
// Change replication
LOG.info("Changing replication to 1");
fs.setReplication(TEST_FILE_PATH, (short)1);
+ BlockManagerTestUtil.computeAllPendingWork(
+ nn1.getNamesystem().getBlockManager());
waitForBlockLocations(cluster, nn1, TEST_FILE, 1);
nn1.getRpcServer().rollEditLog();
@@ -121,6 +122,8 @@ public class TestStandbyIsHot {
// Change back to 3
LOG.info("Changing replication to 3");
fs.setReplication(TEST_FILE_PATH, (short)3);
+ BlockManagerTestUtil.computeAllPendingWork(
+ nn1.getNamesystem().getBlockManager());
nn1.getRpcServer().rollEditLog();
LOG.info("Waiting for higher replication to show up on standby");
@@ -142,7 +145,7 @@ public class TestStandbyIsHot {
* In the bug, the standby node would only very slowly notice the blocks returning
* to the cluster.
*/
- @Test
+ @Test(timeout=60000)
public void testDatanodeRestarts() throws Exception {
Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
@@ -224,17 +227,16 @@ public class TestStandbyIsHot {
LOG.info("Got " + numReplicas + " locs: " + locs);
if (numReplicas > expectedReplicas) {
- for (DataNode dn : cluster.getDataNodes()) {
- DataNodeTestUtils.triggerDeletionReport(dn);
- }
+ cluster.triggerDeletionReports();
}
+ cluster.triggerHeartbeats();
return numReplicas == expectedReplicas;
} catch (IOException e) {
LOG.warn("No block locations yet: " + e.getMessage());
return false;
}
}
- }, 500, 10000);
+ }, 500, 20000);
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java Wed Jun 6 00:17:38 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.tools;
import static org.junit.Assert.*;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
@@ -32,14 +33,17 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
+import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HAServiceTarget;
import org.apache.hadoop.ha.HealthCheckFailedException;
-import org.apache.hadoop.ha.NodeFencer;
+import org.apache.hadoop.ha.ZKFCProtocol;
import org.apache.hadoop.test.MockitoUtil;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import com.google.common.base.Charsets;
@@ -52,6 +56,7 @@ public class TestDFSHAAdmin {
private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream();
private String errOutput;
private HAServiceProtocol mockProtocol;
+ private ZKFCProtocol mockZkfcProtocol;
private static final String NSID = "ns1";
@@ -59,13 +64,16 @@ public class TestDFSHAAdmin {
new HAServiceStatus(HAServiceState.STANDBY)
.setReadyToBecomeActive();
+ private ArgumentCaptor<StateChangeRequestInfo> reqInfoCaptor =
+ ArgumentCaptor.forClass(StateChangeRequestInfo.class);
+
private static String HOST_A = "1.2.3.1";
private static String HOST_B = "1.2.3.2";
private HdfsConfiguration getHAConf() {
HdfsConfiguration conf = new HdfsConfiguration();
- conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, NSID);
- conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID, NSID);
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID);
+ conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, NSID);
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID), "nn1,nn2");
conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1");
@@ -81,6 +89,7 @@ public class TestDFSHAAdmin {
@Before
public void setup() throws IOException {
mockProtocol = MockitoUtil.mockProtocol(HAServiceProtocol.class);
+ mockZkfcProtocol = MockitoUtil.mockProtocol(ZKFCProtocol.class);
tool = new DFSHAAdmin() {
@Override
@@ -90,7 +99,9 @@ public class TestDFSHAAdmin {
// OVerride the target to return our mock protocol
try {
Mockito.doReturn(mockProtocol).when(spy).getProxy(
- Mockito.<Configuration>any(), Mockito.anyInt());
+ Mockito.<Configuration>any(), Mockito.anyInt());
+ Mockito.doReturn(mockZkfcProtocol).when(spy).getZKFCProxy(
+ Mockito.<Configuration>any(), Mockito.anyInt());
} catch (IOException e) {
throw new AssertionError(e); // mock setup doesn't really throw
}
@@ -139,13 +150,89 @@ public class TestDFSHAAdmin {
@Test
public void testTransitionToActive() throws Exception {
assertEquals(0, runTool("-transitionToActive", "nn1"));
- Mockito.verify(mockProtocol).transitionToActive();
+ Mockito.verify(mockProtocol).transitionToActive(
+ reqInfoCaptor.capture());
+ assertEquals(RequestSource.REQUEST_BY_USER,
+ reqInfoCaptor.getValue().getSource());
+ }
+
+ /**
+ * Test that, if automatic HA is enabled, none of the mutative operations
+ * will succeed, unless the -forcemanual flag is specified.
+ * @throws Exception
+ */
+ @Test
+ public void testMutativeOperationsWithAutoHaEnabled() throws Exception {
+ Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
+
+ // Turn on auto-HA in the config
+ HdfsConfiguration conf = getHAConf();
+ conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ tool.setConf(conf);
+
+ // Should fail without the forcemanual flag
+ assertEquals(-1, runTool("-transitionToActive", "nn1"));
+ assertTrue(errOutput.contains("Refusing to manually manage"));
+ assertEquals(-1, runTool("-transitionToStandby", "nn1"));
+ assertTrue(errOutput.contains("Refusing to manually manage"));
+
+ Mockito.verify(mockProtocol, Mockito.never())
+ .transitionToActive(anyReqInfo());
+ Mockito.verify(mockProtocol, Mockito.never())
+ .transitionToStandby(anyReqInfo());
+
+ // Force flag should bypass the check and change the request source
+ // for the RPC
+ setupConfirmationOnSystemIn();
+ assertEquals(0, runTool("-transitionToActive", "-forcemanual", "nn1"));
+ setupConfirmationOnSystemIn();
+ assertEquals(0, runTool("-transitionToStandby", "-forcemanual", "nn1"));
+
+ Mockito.verify(mockProtocol, Mockito.times(1)).transitionToActive(
+ reqInfoCaptor.capture());
+ Mockito.verify(mockProtocol, Mockito.times(1)).transitionToStandby(
+ reqInfoCaptor.capture());
+
+ // All of the RPCs should have had the "force" source
+ for (StateChangeRequestInfo ri : reqInfoCaptor.getAllValues()) {
+ assertEquals(RequestSource.REQUEST_BY_USER_FORCED, ri.getSource());
+ }
+ }
+
+ /**
+ * Setup System.in with a stream that feeds a "yes" answer on the
+ * next prompt.
+ */
+ private static void setupConfirmationOnSystemIn() {
+ // Answer "yes" to the prompt about transition to active
+ System.setIn(new ByteArrayInputStream("yes\n".getBytes()));
+ }
+
+ /**
+ * Test that, even if automatic HA is enabled, the monitoring operations
+ * still function correctly.
+ */
+ @Test
+ public void testMonitoringOperationsWithAutoHaEnabled() throws Exception {
+ Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
+
+ // Turn on auto-HA
+ HdfsConfiguration conf = getHAConf();
+ conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
+ tool.setConf(conf);
+
+ assertEquals(0, runTool("-checkHealth", "nn1"));
+ Mockito.verify(mockProtocol).monitorHealth();
+
+ assertEquals(0, runTool("-getServiceState", "nn1"));
+ Mockito.verify(mockProtocol).getServiceStatus();
}
@Test
public void testTransitionToStandby() throws Exception {
assertEquals(0, runTool("-transitionToStandby", "nn1"));
- Mockito.verify(mockProtocol).transitionToStandby();
+ Mockito.verify(mockProtocol).transitionToStandby(anyReqInfo());
}
@Test
@@ -213,6 +300,19 @@ public class TestDFSHAAdmin {
tool.setConf(conf);
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
}
+
+ @Test
+ public void testFailoverWithAutoHa() throws Exception {
+ Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
+ // Turn on auto-HA in the config
+ HdfsConfiguration conf = getHAConf();
+ conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true);
+ conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, "shell(true)");
+ tool.setConf(conf);
+
+ assertEquals(0, runTool("-failover", "nn1", "nn2"));
+ Mockito.verify(mockZkfcProtocol).gracefulFailover();
+ }
@Test
public void testForceFenceOptionListedBeforeArgs() throws Exception {
@@ -283,4 +383,8 @@ public class TestDFSHAAdmin {
LOG.info("Output:\n" + errOutput);
return ret;
}
+
+ private StateChangeRequestInfo anyReqInfo() {
+ return Mockito.<StateChangeRequestInfo>any();
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java Wed Jun 6 00:17:38 2012
@@ -61,7 +61,7 @@ public class TestGetConf {
}
nsList.append(getNameServiceId(i));
}
- conf.set(DFS_FEDERATION_NAMESERVICES, nsList.toString());
+ conf.set(DFS_NAMESERVICES, nsList.toString());
}
/** Set a given key with value as address, for all the nameServiceIds.
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Wed Jun 6 00:17:38 2012
@@ -18,6 +18,10 @@
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import java.io.BufferedReader;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.token.Token;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
@@ -29,15 +33,19 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Set;
-import junit.framework.TestCase;
+import org.junit.*;
+import static org.junit.Assert.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
@@ -52,12 +60,15 @@ import org.apache.hadoop.hdfs.HdfsConfig
* * confirm it correctly bails on malformed image files, in particular, a
* file that ends suddenly.
*/
-public class TestOfflineImageViewer extends TestCase {
+public class TestOfflineImageViewer {
+ private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
private static final int NUM_DIRS = 3;
private static final int FILES_PER_DIR = 4;
+ private static final String TEST_RENEWER = "JobTracker";
+ private static File originalFsimage = null;
// Elements of lines of ls-file output to be compared to FileStatus instance
- private class LsElements {
+ private static class LsElements {
public String perms;
public int replication;
public String username;
@@ -67,43 +78,28 @@ public class TestOfflineImageViewer exte
}
// namespace as written to dfs, to be compared with viewer's output
- final HashMap<String, FileStatus> writtenFiles
- = new HashMap<String, FileStatus>();
-
+ final static HashMap<String, FileStatus> writtenFiles =
+ new HashMap<String, FileStatus>();
private static String ROOT = System.getProperty("test.build.data",
"build/test/data");
- // Main entry point into testing. Necessary since we only want to generate
- // the fsimage file once and use it for multiple tests.
- public void testOIV() throws Exception {
- File originalFsimage = null;
- try {
- originalFsimage = initFsimage();
- assertNotNull("originalFsImage shouldn't be null", originalFsimage);
-
- // Tests:
- outputOfLSVisitor(originalFsimage);
- outputOfFileDistributionVisitor(originalFsimage);
-
- unsupportedFSLayoutVersion(originalFsimage);
-
- truncatedFSImage(originalFsimage);
-
- } finally {
- if(originalFsimage != null && originalFsimage.exists())
- originalFsimage.delete();
- }
- }
-
// Create a populated namespace for later testing. Save its contents to a
// data structure and store its fsimage location.
- private File initFsimage() throws IOException {
+ // We only want to generate the fsimage file once and use it for
+ // multiple tests.
+ @BeforeClass
+ public static void createOriginalFSImage() throws IOException {
MiniDFSCluster cluster = null;
- File orig = null;
try {
Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ conf.set("hadoop.security.auth_to_local",
+ "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+ cluster.waitActive();
FileSystem hdfs = cluster.getFileSystem();
int filesize = 256;
@@ -123,34 +119,49 @@ public class TestOfflineImageViewer exte
}
}
+ // Get delegation tokens so we log the delegation token op
+ List<Token<?>> delegationTokens =
+ hdfs.getDelegationTokens(TEST_RENEWER);
+ for (Token<?> t : delegationTokens) {
+ LOG.debug("got token " + t);
+ }
+
// Write results to the fsimage file
cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
cluster.getNameNodeRpc().saveNamespace();
// Determine location of fsimage file
- orig = FSImageTestUtil.findLatestImageFile(
+ originalFsimage = FSImageTestUtil.findLatestImageFile(
FSImageTestUtil.getFSImage(
cluster.getNameNode()).getStorage().getStorageDir(0));
- if (orig == null) {
- fail("Didn't generate or can't find fsimage");
+ if (originalFsimage == null) {
+ throw new RuntimeException("Didn't generate or can't find fsimage");
}
+ LOG.debug("original FS image file is " + originalFsimage);
} finally {
if(cluster != null)
cluster.shutdown();
}
- return orig;
+ }
+
+ @AfterClass
+ public static void deleteOriginalFSImage() throws IOException {
+ if(originalFsimage != null && originalFsimage.exists()) {
+ originalFsimage.delete();
+ }
}
// Convenience method to generate a file status from file system for
// later comparison
- private FileStatus pathToFileEntry(FileSystem hdfs, String file)
+ private static FileStatus pathToFileEntry(FileSystem hdfs, String file)
throws IOException {
return hdfs.getFileStatus(new Path(file));
}
-
+
// Verify that we can correctly generate an ls-style output for a valid
// fsimage
- private void outputOfLSVisitor(File originalFsimage) throws IOException {
+ @Test
+ public void outputOfLSVisitor() throws IOException {
File testFile = new File(ROOT, "/basicCheck");
File outputFile = new File(ROOT, "/basicCheckOutput");
@@ -169,12 +180,13 @@ public class TestOfflineImageViewer exte
if(testFile.exists()) testFile.delete();
if(outputFile.exists()) outputFile.delete();
}
- System.out.println("Correctly generated ls-style output.");
+ LOG.debug("Correctly generated ls-style output.");
}
// Confirm that attempting to read an fsimage file with an unsupported
// layout results in an error
- public void unsupportedFSLayoutVersion(File originalFsimage) throws IOException {
+ @Test
+ public void unsupportedFSLayoutVersion() throws IOException {
File testFile = new File(ROOT, "/invalidLayoutVersion");
File outputFile = new File(ROOT, "invalidLayoutVersionOutput");
@@ -190,7 +202,7 @@ public class TestOfflineImageViewer exte
} catch(IOException e) {
if(!e.getMessage().contains(Integer.toString(badVersionNum)))
throw e; // wasn't error we were expecting
- System.out.println("Correctly failed at reading bad image version.");
+ LOG.debug("Correctly failed at reading bad image version.");
}
} finally {
if(testFile.exists()) testFile.delete();
@@ -199,7 +211,8 @@ public class TestOfflineImageViewer exte
}
// Verify that image viewer will bail on a file that ends unexpectedly
- private void truncatedFSImage(File originalFsimage) throws IOException {
+ @Test
+ public void truncatedFSImage() throws IOException {
File testFile = new File(ROOT, "/truncatedFSImage");
File outputFile = new File(ROOT, "/trucnatedFSImageOutput");
try {
@@ -213,7 +226,7 @@ public class TestOfflineImageViewer exte
oiv.go();
fail("Managed to process a truncated fsimage file");
} catch (EOFException e) {
- System.out.println("Correctly handled EOF");
+ LOG.debug("Correctly handled EOF");
}
} finally {
@@ -365,7 +378,8 @@ public class TestOfflineImageViewer exte
}
}
- private void outputOfFileDistributionVisitor(File originalFsimage) throws IOException {
+ @Test
+ public void outputOfFileDistributionVisitor() throws IOException {
File testFile = new File(ROOT, "/basicCheck");
File outputFile = new File(ROOT, "/fileDistributionCheckOutput");
@@ -392,4 +406,66 @@ public class TestOfflineImageViewer exte
}
assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR);
}
+
+ private static class TestImageVisitor extends ImageVisitor {
+ private List<String> delegationTokenRenewers = new LinkedList<String>();
+ TestImageVisitor() {
+ }
+
+ List<String> getDelegationTokenRenewers() {
+ return delegationTokenRenewers;
+ }
+
+ @Override
+ void start() throws IOException {
+ }
+
+ @Override
+ void finish() throws IOException {
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) {
+ delegationTokenRenewers.add(value);
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element, ImageElement key,
+ String value) throws IOException {
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ }
+ }
+
+ @Test
+ public void outputOfTestVisitor() throws IOException {
+ File testFile = new File(ROOT, "/basicCheck");
+
+ try {
+ copyFile(originalFsimage, testFile);
+ TestImageVisitor v = new TestImageVisitor();
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true);
+ oiv.go();
+
+ // Validated stored delegation token identifiers.
+ List<String> dtrs = v.getDelegationTokenRenewers();
+ assertEquals(1, dtrs.size());
+ assertEquals(TEST_RENEWER, dtrs.get(0));
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ }
+ LOG.debug("Passed TestVisitor validation.");
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java Wed Jun 6 00:17:38 2012
@@ -421,5 +421,48 @@ public class TestLightWeightHashSet{
LOG.info("Test other - DONE");
}
+
+ @Test
+ public void testGetElement() {
+ LightWeightHashSet<TestObject> objSet = new LightWeightHashSet<TestObject>();
+ TestObject objA = new TestObject("object A");
+ TestObject equalToObjA = new TestObject("object A");
+ TestObject objB = new TestObject("object B");
+ objSet.add(objA);
+ objSet.add(objB);
+
+ assertSame(objA, objSet.getElement(objA));
+ assertSame(objA, objSet.getElement(equalToObjA));
+ assertSame(objB, objSet.getElement(objB));
+ assertNull(objSet.getElement(new TestObject("not in set")));
+ }
+
+ /**
+ * Wrapper class which is used in
+ * {@link TestLightWeightHashSet#testGetElement()}
+ */
+ private static class TestObject {
+ private final String value;
+
+ public TestObject(String value) {
+ super();
+ this.value = value;
+ }
+
+ @Override
+ public int hashCode() {
+ return value.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass())
+ return false;
+ TestObject other = (TestObject) obj;
+ return this.value.equals(other.value);
+ }
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java Wed Jun 6 00:17:38 2012
@@ -123,9 +123,10 @@ public class TestPermission extends Test
checkPermission(fs, "/aa/1/aa/2/aa/3", dirPerm);
FsPermission filePerm = new FsPermission((short)0444);
- FSDataOutputStream out = fs.create(new Path("/b1/b2/b3.txt"), filePerm,
+ Path p = new Path("/b1/b2/b3.txt");
+ FSDataOutputStream out = fs.create(p, filePerm,
true, conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
+ fs.getDefaultReplication(p), fs.getDefaultBlockSize(p), null);
out.write(123);
out.close();
checkPermission(fs, "/b1", inheritPerm);
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-policy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-policy.xml?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-policy.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-policy.xml Wed Jun 6 00:17:38 2012
@@ -116,5 +116,11 @@
<description>ACL for HAService protocol used by HAAdmin to manage the
active and stand-by states of namenode.</description>
</property>
+ <property>
+ <name>security.zkfc.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for access to the ZK Failover Controller
+ </description>
+ </property>
</configuration>