You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2016/10/21 02:52:43 UTC
hadoop git commit: HDFS-10998. Add unit tests for HDFS command
'dfsadmin -fetchImage' in HA. Contributed by Xiaobing Zhou
Repository: hadoop
Updated Branches:
refs/heads/trunk 262827cf7 -> d7d87deec
HDFS-10998. Add unit tests for HDFS command 'dfsadmin -fetchImage' in HA. Contributed by Xiaobing Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7d87dee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7d87dee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7d87dee
Branch: refs/heads/trunk
Commit: d7d87deece66333c188e9b7c10b4b56ddb529ce9
Parents: 262827c
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Oct 20 19:51:48 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Oct 20 19:51:48 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/TestFetchImage.java | 105 ++++++++++++++-----
1 file changed, 79 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7d87dee/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java
index d8218b6..7e1e593 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java
@@ -17,10 +17,15 @@
*/
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
import static org.junit.Assert.assertEquals;
import java.io.File;
+import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -29,11 +34,16 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.PathUtils;
import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestFetchImage {
@@ -43,46 +53,89 @@ public class TestFetchImage {
// Shamelessly stolen from NNStorage.
private static final Pattern IMAGE_REGEX = Pattern.compile("fsimage_(\\d+)");
+ private MiniDFSCluster cluster;
+ private NameNode nn0 = null;
+ private NameNode nn1 = null;
+ private Configuration conf = null;
+
+ @BeforeClass
+ public static void setupImageDir() {
+ FETCHED_IMAGE_FILE.mkdirs();
+ }
+
@AfterClass
public static void cleanup() {
FileUtil.fullyDelete(FETCHED_IMAGE_FILE);
}
+ @Before
+ public void setupCluster() throws IOException, URISyntaxException {
+ conf = new Configuration();
+ conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ conf.setLong(DFS_BLOCK_SIZE_KEY, 1024);
+
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(1)
+ .build();
+ nn0 = cluster.getNameNode(0);
+ nn1 = cluster.getNameNode(1);
+ HATestUtil.configureFailoverFs(cluster, conf);
+ cluster.waitActive();
+ }
+
/**
* Download a few fsimages using `hdfs dfsadmin -fetchImage ...' and verify
* the results.
*/
- @Test
- public void testFetchImage() throws Exception {
- FETCHED_IMAGE_FILE.mkdirs();
- Configuration conf = new Configuration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- FileSystem fs = null;
- try {
- DFSAdmin dfsAdmin = new DFSAdmin();
- dfsAdmin.setConf(conf);
-
+ @Test(timeout=30000)
+ public void testFetchImageHA() throws Exception {
+ final Path parent = new Path(
+ PathUtils.getTestPath(getClass()),
+ GenericTestUtils.getMethodName());
+
+ int nnIndex = 0;
+ /* run on nn0 as active */
+ cluster.transitionToActive(nnIndex);
+ testFetchImageInternal(
+ nnIndex,
+ new Path(parent, "dir1"),
+ new Path(parent, "dir2"));
+
+ /* run on nn1 as active */
+ nnIndex = 1;
+ HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
+ cluster.transitionToActive(nnIndex);
+ testFetchImageInternal(
+ nnIndex,
+ new Path(parent, "dir3"),
+ new Path(parent, "dir4"));
+ }
+
+ private void testFetchImageInternal(
+ final int nnIndex,
+ final Path dir1,
+ final Path dir2) throws Exception {
+ final Configuration dfsConf = cluster.getConfiguration(nnIndex);
+ final DFSAdmin dfsAdmin = new DFSAdmin();
+ dfsAdmin.setConf(dfsConf);
+
+ try (FileSystem fs = cluster.getFileSystem(nnIndex)) {
runFetchImage(dfsAdmin, cluster);
- fs = cluster.getFileSystem();
- fs.mkdirs(new Path("/foo"));
- fs.mkdirs(new Path("/foo2"));
- fs.mkdirs(new Path("/foo3"));
+ fs.mkdirs(dir1);
+ fs.mkdirs(dir2);
- cluster.getNameNodeRpc()
- .setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
- cluster.getNameNodeRpc().saveNamespace(0, 0);
- cluster.getNameNodeRpc()
- .setSafeMode(SafeModeAction.SAFEMODE_LEAVE, false);
+ cluster.getNameNodeRpc(nnIndex).setSafeMode(
+ SafeModeAction.SAFEMODE_ENTER,
+ false);
+ cluster.getNameNodeRpc(nnIndex).saveNamespace(0, 0);
+ cluster.getNameNodeRpc(nnIndex).setSafeMode(
+ SafeModeAction.SAFEMODE_LEAVE,
+ false);
runFetchImage(dfsAdmin, cluster);
- } finally {
- if (fs != null) {
- fs.close();
- }
- if (cluster != null) {
- cluster.shutdown();
- }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org