You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/04/12 22:52:41 UTC
[16/50] hadoop git commit: HDFS-11633. FSImage failover disables all
erasure coding policies. Contributed by Wei-Chiu Chuang.
HDFS-11633. FSImage failover disables all erasure coding policies. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/203edc02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/203edc02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/203edc02
Branch: refs/heads/HADOOP-13345
Commit: 203edc026c39224d1a5f762d05594c3757c6d908
Parents: 63f7322
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Apr 7 15:46:11 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Apr 7 15:46:11 2017 -0700
----------------------------------------------------------------------
.../namenode/ErasureCodingPolicyManager.java | 3 +-
.../hdfs/server/namenode/TestStartup.java | 57 +++++++++++++++++++-
2 files changed, 58 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/203edc02/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index c23b034..17b48f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -190,6 +190,7 @@ public final class ErasureCodingPolicyManager {
* Clear and clean up.
*/
public void clear() {
- enabledPoliciesByName.clear();
+ // TODO: we should only clear policies loaded from NN metadata.
+ // This is a placeholder for HDFS-7337.
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/203edc02/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
index 5da19a7..8c2acf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
@@ -41,6 +41,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -48,10 +49,13 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.LogVerificationAppender;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
@@ -564,7 +568,58 @@ public class TestStartup {
} finally {
cluster.shutdown();
}
-}
+ }
+
+ @Test(timeout=30000)
+ public void testCorruptImageFallbackLostECPolicy() throws IOException {
+ final ErasureCodingPolicy defaultPolicy = StripedFileTestUtil
+ .getDefaultECPolicy();
+ final String policy = defaultPolicy.getName();
+ final Path f1 = new Path("/f1");
+ config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, policy);
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(config)
+ .numDataNodes(0)
+ .format(true)
+ .build();
+ try {
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ // set root directory to use the default ec policy
+ Path srcECDir = new Path("/");
+ fs.setErasureCodingPolicy(srcECDir,
+ defaultPolicy.getName());
+
+ // create a file which will use the default ec policy
+ fs.create(f1);
+ FileStatus fs1 = fs.getFileStatus(f1);
+ assertTrue(fs1.isErasureCoded());
+ ErasureCodingPolicy fs1Policy = fs.getErasureCodingPolicy(f1);
+ assertEquals(fs1Policy, defaultPolicy);
+ } finally {
+ cluster.close();
+ }
+
+ // Delete a single md5sum
+ corruptFSImageMD5(false);
+ // Should still be able to start
+ cluster = new MiniDFSCluster.Builder(config)
+ .numDataNodes(0)
+ .format(false)
+ .build();
+ try {
+ cluster.waitActive();
+ ErasureCodingPolicy[] ecPolicies = cluster.getNameNode()
+ .getNamesystem().getErasureCodingPolicyManager().getEnabledPolicies();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ // make sure the ec policy of the file is still correct
+ assertEquals(fs.getErasureCodingPolicy(f1), defaultPolicy);
+ // make sure after fsimage fallback, enabled ec policies are not cleared.
+ assertTrue(ecPolicies.length == 1);
+ } finally {
+ cluster.shutdown();
+ }
+ }
/**
* This test tests hosts include list contains host names. After namenode
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org