You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xi...@apache.org on 2018/11/08 18:01:20 UTC
hadoop git commit: HDFS-14039. ec -listPolicies doesn't show correct
state for the default policy when the default is not RS(6,
3). Contributed by Kitti Nanasi.
Repository: hadoop
Updated Branches:
refs/heads/trunk 724c15007 -> 8d99648c2
HDFS-14039. ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6,3). Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen <xi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d99648c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d99648c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d99648c
Branch: refs/heads/trunk
Commit: 8d99648c203004045a9339ad27258092969145d6
Parents: 724c150
Author: Kitti Nanasi <kn...@cloudera.com>
Authored: Thu Nov 8 10:00:09 2018 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Nov 8 10:01:19 2018 -0800
----------------------------------------------------------------------
.../namenode/ErasureCodingPolicyManager.java | 119 ++++++++++++++-----
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../server/namenode/TestEnabledECPolicies.java | 103 +++++++++++++++-
.../hdfs/server/namenode/TestFSImage.java | 42 +++++--
.../server/namenode/TestNamenodeRetryCache.java | 2 +-
.../server/namenode/TestStripedINodeFile.java | 2 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
8 files changed, 231 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index d2bf3af..57fa958 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -81,6 +82,15 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyInfo[] allPolicies;
/**
+ * All policies in the state as it will be persisted in the fsimage.
+ *
+ * The difference between persisted policies and all policies is that
+ * if a default policy is only enabled at startup,
+ * it will appear as disabled in the persisted policy list and in the fsimage.
+ */
+ private Map<Byte, ErasureCodingPolicyInfo> allPersistedPolicies;
+
+ /**
* All enabled policies sorted by name for fast querying, including built-in
* policy, user defined policy.
*/
@@ -90,6 +100,7 @@ public final class ErasureCodingPolicyManager {
*/
private ErasureCodingPolicy[] enabledPolicies;
+ private String defaultPolicyName;
private volatile static ErasureCodingPolicyManager instance = null;
@@ -102,14 +113,11 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyManager() {}
- public void init(Configuration conf) {
- // Load erasure coding default policy
- final String defaultPolicyName = conf.getTrimmed(
- DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
- DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ public void init(Configuration conf) throws IOException {
this.policiesByName = new TreeMap<>();
this.policiesByID = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
+ this.allPersistedPolicies = new TreeMap<>();
/**
* TODO: load user defined EC policy from fsImage HDFS-7859
@@ -125,31 +133,12 @@ public final class ErasureCodingPolicyManager {
final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
policiesByName.put(policy.getName(), info);
policiesByID.put(policy.getId(), info);
+ allPersistedPolicies.put(policy.getId(),
+ new ErasureCodingPolicyInfo(policy));
}
- if (!defaultPolicyName.isEmpty()) {
- final ErasureCodingPolicyInfo info =
- policiesByName.get(defaultPolicyName);
- if (info == null) {
- String names = policiesByName.values()
- .stream().map((pi) -> pi.getPolicy().getName())
- .collect(Collectors.joining(", "));
- String msg = String.format("EC policy '%s' specified at %s is not a "
- + "valid policy. Please choose from list of available "
- + "policies: [%s]",
- defaultPolicyName,
- DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
- names);
- throw new HadoopIllegalArgumentException(msg);
- }
- info.setState(ErasureCodingPolicyState.ENABLED);
- enabledPoliciesByName.put(info.getPolicy().getName(), info.getPolicy());
- }
- enabledPolicies =
- enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
- allPolicies =
- policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]);
-
+ enableDefaultPolicy(conf);
+ updatePolicies();
maxCellSize = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_KEY,
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT);
@@ -201,6 +190,21 @@ public final class ErasureCodingPolicyManager {
}
/**
+ * Get all system defined policies and user defined policies
+ * as it is written out in the fsimage.
+ *
+ * The difference between persisted policies and all policies is that
+ * if a default policy is only enabled at startup,
+ * it will appear as disabled in the persisted policy list and in the fsimage.
+ *
+ * @return persisted policies
+ */
+ public ErasureCodingPolicyInfo[] getPersistedPolicies() {
+ return allPersistedPolicies.values()
+ .toArray(new ErasureCodingPolicyInfo[0]);
+ }
+
+ /**
* Get a {@link ErasureCodingPolicy} by policy ID, including system policy
* and user defined policy.
* @return ecPolicy, or null if not found
@@ -299,6 +303,8 @@ public final class ErasureCodingPolicyManager {
this.policiesByID.put(policy.getId(), pi);
allPolicies =
policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]);
+ allPersistedPolicies.put(policy.getId(),
+ new ErasureCodingPolicyInfo(policy));
return policy;
}
@@ -335,7 +341,8 @@ public final class ErasureCodingPolicyManager {
}
info.setState(ErasureCodingPolicyState.REMOVED);
LOG.info("Remove erasure coding policy " + name);
-
+ allPersistedPolicies.put(ecPolicy.getId(),
+ createPolicyInfo(ecPolicy, ErasureCodingPolicyState.REMOVED));
/*
* TODO HDFS-12405 postpone the delete removed policy to Namenode restart
* time.
@@ -370,6 +377,9 @@ public final class ErasureCodingPolicyManager {
enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
info.setState(ErasureCodingPolicyState.DISABLED);
LOG.info("Disable the erasure coding policy " + name);
+ allPersistedPolicies.put(info.getPolicy().getId(),
+ createPolicyInfo(info.getPolicy(),
+ ErasureCodingPolicyState.DISABLED));
return true;
}
return false;
@@ -385,6 +395,12 @@ public final class ErasureCodingPolicyManager {
name + " does not exist");
}
if (enabledPoliciesByName.containsKey(name)) {
+ if (defaultPolicyName.equals(name)) {
+ allPersistedPolicies.put(info.getPolicy().getId(),
+ createPolicyInfo(info.getPolicy(),
+ ErasureCodingPolicyState.ENABLED));
+ return true;
+ }
return false;
}
final ErasureCodingPolicy ecPolicy = info.getPolicy();
@@ -392,6 +408,8 @@ public final class ErasureCodingPolicyManager {
info.setState(ErasureCodingPolicyState.ENABLED);
enabledPolicies =
enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
+ allPersistedPolicies.put(ecPolicy.getId(),
+ createPolicyInfo(info.getPolicy(), ErasureCodingPolicyState.ENABLED));
LOG.info("Enable the erasure coding policy " + name);
return true;
}
@@ -414,6 +432,8 @@ public final class ErasureCodingPolicyManager {
if (info.isEnabled()) {
enablePolicy(policy.getName());
}
+ allPersistedPolicies.put(policy.getId(),
+ createPolicyInfo(policy, info.getState()));
}
/**
@@ -423,11 +443,43 @@ public final class ErasureCodingPolicyManager {
*
*/
public synchronized void loadPolicies(
- List<ErasureCodingPolicyInfo> ecPolicies) {
+ List<ErasureCodingPolicyInfo> ecPolicies, Configuration conf)
+ throws IOException{
Preconditions.checkNotNull(ecPolicies);
for (ErasureCodingPolicyInfo p : ecPolicies) {
loadPolicy(p);
}
+ enableDefaultPolicy(conf);
+ updatePolicies();
+ }
+
+ private void enableDefaultPolicy(Configuration conf) throws IOException {
+ defaultPolicyName = conf.getTrimmed(
+ DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
+ DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ if (!defaultPolicyName.isEmpty()) {
+ final ErasureCodingPolicyInfo info =
+ policiesByName.get(defaultPolicyName);
+ if (info == null) {
+ String names = policiesByName.values()
+ .stream().map((pi) -> pi.getPolicy().getName())
+ .collect(Collectors.joining(", "));
+ String msg = String.format("EC policy '%s' specified at %s is not a "
+ + "valid policy. Please choose from list of available "
+ + "policies: [%s]",
+ defaultPolicyName,
+ DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
+ names);
+ throw new IOException(msg);
+ }
+ info.setState(ErasureCodingPolicyState.ENABLED);
+ enabledPoliciesByName.put(info.getPolicy().getName(), info.getPolicy());
+ }
+ }
+
+ private void updatePolicies() {
+ enabledPolicies =
+ enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
allPolicies =
policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]);
}
@@ -436,4 +488,11 @@ public final class ErasureCodingPolicyManager {
return StringUtils.join(", ",
enabledPoliciesByName.keySet());
}
+
+ private ErasureCodingPolicyInfo createPolicyInfo(ErasureCodingPolicy p,
+ ErasureCodingPolicyState s) {
+ ErasureCodingPolicyInfo policyInfo = new ErasureCodingPolicyInfo(p);
+ policyInfo.setState(s);
+ return policyInfo;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index 95eade9..7aed5fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -380,7 +380,7 @@ public final class FSImageFormatProtobuf {
ecPolicies.add(PBHelperClient.convertErasureCodingPolicyInfo(
s.getPolicies(i)));
}
- fsn.getErasureCodingPolicyManager().loadPolicies(ecPolicies);
+ fsn.getErasureCodingPolicyManager().loadPolicies(ecPolicies, conf);
}
}
@@ -601,7 +601,7 @@ public final class FSImageFormatProtobuf {
FileSummary.Builder summary) throws IOException {
final FSNamesystem fsn = context.getSourceNamesystem();
ErasureCodingPolicyInfo[] ecPolicies =
- fsn.getErasureCodingPolicyManager().getPolicies();
+ fsn.getErasureCodingPolicyManager().getPersistedPolicies();
ArrayList<ErasureCodingPolicyProto> ecPolicyProtoes =
new ArrayList<ErasureCodingPolicyProto>();
for (ErasureCodingPolicyInfo p : ecPolicies) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 0bef4cc..c7e5147 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -503,7 +503,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY,
DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT);
- defaultECPolicyName = conf.get(
+ defaultECPolicyName = conf.getTrimmed(
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java
index 63bfa27..843e9e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.test.GenericTestUtils;
@@ -28,10 +30,16 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
@@ -43,14 +51,14 @@ public class TestEnabledECPolicies {
@Rule
public Timeout testTimeout = new Timeout(60000);
- private void expectInvalidPolicy(String value) {
+ private void expectInvalidPolicy(String value) throws IOException {
HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
value);
try {
ErasureCodingPolicyManager.getInstance().init(conf);
fail("Expected exception when instantiating ECPolicyManager");
- } catch (IllegalArgumentException e) {
+ } catch (IOException e) {
GenericTestUtils.assertExceptionContains("is not a valid policy", e);
}
}
@@ -117,6 +125,70 @@ public class TestEnabledECPolicies {
testGetPolicies(enabledPolicies);
}
+ @Test
+ public void testChangeDefaultPolicy() throws Exception {
+ final HdfsConfiguration conf = new HdfsConfiguration();
+ final String testPolicy = "RS-3-2-1024k";
+ final String defaultPolicy = conf.getTrimmed(
+ DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
+ DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ assertNotEquals("The default policy and the next default policy " +
+ "should not be the same!", testPolicy, defaultPolicy);
+
+ ErasureCodingPolicyManager manager =
+ ErasureCodingPolicyManager.getInstance();
+ // Change the default policy to a new one
+ conf.set(
+ DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
+ testPolicy);
+ manager.init(conf);
+ // Load policies similar to when fsimage is loaded at namenode startup
+ manager.loadPolicies(constructAllDisabledInitialPolicies(), conf);
+
+ ErasureCodingPolicyInfo[] getPoliciesResult = manager.getPolicies();
+ boolean isEnabled = isPolicyEnabled(testPolicy, getPoliciesResult);
+ assertTrue("The new default policy should be " +
+ "in enabled state!", isEnabled);
+ ErasureCodingPolicyInfo[] getPersistedPoliciesResult
+ = manager.getPersistedPolicies();
+ isEnabled = isPolicyEnabled(testPolicy, getPersistedPoliciesResult);
+ assertFalse("The new default policy should be " +
+ "in disabled state in the persisted list!", isEnabled);
+
+ manager.disablePolicy(testPolicy);
+ getPoliciesResult = manager.getPolicies();
+ isEnabled = isPolicyEnabled(testPolicy, getPoliciesResult);
+ assertFalse("The new default policy should be " +
+ "in disabled state!", isEnabled);
+ getPersistedPoliciesResult
+ = manager.getPersistedPolicies();
+ isEnabled = isPolicyEnabled(testPolicy, getPersistedPoliciesResult);
+ assertFalse("The new default policy should be " +
+ "in disabled state in the persisted list!", isEnabled);
+
+ manager.enablePolicy(testPolicy);
+ getPoliciesResult = manager.getPolicies();
+ isEnabled = isPolicyEnabled(testPolicy, getPoliciesResult);
+ assertTrue("The new default policy should be " +
+ "in enabled state!", isEnabled);
+ getPersistedPoliciesResult
+ = manager.getPersistedPolicies();
+ isEnabled = isPolicyEnabled(testPolicy, getPersistedPoliciesResult);
+ assertTrue("The new default policy should be " +
+ "in enabled state in the persisted list!", isEnabled);
+
+ final String emptyPolicy = "";
+ // Change the default policy to a empty
+ conf.set(
+ DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY, emptyPolicy);
+ manager.init(conf);
+ // Load policies similar to when fsimage is loaded at namenode startup
+ manager.loadPolicies(constructAllDisabledInitialPolicies(), conf);
+ // All the policies are disabled if the default policy is empty
+ getPoliciesResult = manager.getPolicies();
+ assertAllPoliciesAreDisabled(getPoliciesResult);
+ }
+
private void testGetPolicies(ErasureCodingPolicy[] enabledPolicies)
throws Exception {
HdfsConfiguration conf = new HdfsConfiguration();
@@ -154,4 +226,31 @@ public class TestEnabledECPolicies {
}
}
}
+
+ private List<ErasureCodingPolicyInfo> constructAllDisabledInitialPolicies() {
+ List<ErasureCodingPolicyInfo> policies = new ArrayList<>();
+ for (ErasureCodingPolicy p: SystemErasureCodingPolicies.getPolicies()) {
+ policies.add(new ErasureCodingPolicyInfo(p,
+ ErasureCodingPolicyState.DISABLED));
+ }
+ return policies;
+ }
+
+ private boolean isPolicyEnabled(String testPolicy,
+ ErasureCodingPolicyInfo[] policies) {
+ for (ErasureCodingPolicyInfo p : policies) {
+ if (testPolicy.equals(p.getPolicy().getName())) {
+ return p.isEnabled();
+ }
+ }
+ fail("The result should contain the test policy!");
+ return false;
+ }
+
+ private void assertAllPoliciesAreDisabled(
+ ErasureCodingPolicyInfo[] policies) {
+ for (ErasureCodingPolicyInfo p : policies) {
+ assertTrue("Policy should be disabled", p.isDisabled());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index ba08f73..0beb758 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -37,6 +37,7 @@ import java.util.EnumSet;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -883,15 +884,19 @@ public class TestFSImage {
DFSTestUtil.getECPolicyState(ecPolicy));
// Test enable/disable/remove user customized erasure coding policy
- testChangeErasureCodingPolicyState(cluster, blockSize, newPolicy);
- // Test enable/disable built-in erasure coding policy
+ testChangeErasureCodingPolicyState(cluster, blockSize, newPolicy, false);
+ // Test enable/disable default built-in erasure coding policy
testChangeErasureCodingPolicyState(cluster, blockSize,
- SystemErasureCodingPolicies.getByID((byte) 1));
+ SystemErasureCodingPolicies.getByID((byte) 1), true);
+ // Test enable/disable non-default built-in erasure coding policy
+ testChangeErasureCodingPolicyState(cluster, blockSize,
+ SystemErasureCodingPolicies.getByID((byte) 2), false);
}
}
private void testChangeErasureCodingPolicyState(MiniDFSCluster cluster,
- int blockSize, ErasureCodingPolicy targetPolicy) throws IOException {
+ int blockSize, ErasureCodingPolicy targetPolicy, boolean isDefault)
+ throws IOException {
DistributedFileSystem fs = cluster.getFileSystem();
// 1. Enable an erasure coding policy
@@ -920,6 +925,9 @@ public class TestFSImage {
assertEquals("The erasure coding policy should be of enabled state",
ErasureCodingPolicyState.ENABLED,
DFSTestUtil.getECPolicyState(ecPolicy));
+ assertTrue("Policy should be in disabled state in FSImage!",
+ isPolicyEnabledInFsImage(targetPolicy));
+
// Read file regardless of the erasure coding policy state
DFSTestUtil.readFileAsBytes(fs, filePath);
@@ -936,9 +944,18 @@ public class TestFSImage {
ErasureCodingPolicyManager.getInstance().getByID(targetPolicy.getId());
assertEquals("The erasure coding policy is not found",
targetPolicy, ecPolicy);
- assertEquals("The erasure coding policy should be of disabled state",
- ErasureCodingPolicyState.DISABLED,
- DFSTestUtil.getECPolicyState(ecPolicy));
+ ErasureCodingPolicyState ecPolicyState =
+ DFSTestUtil.getECPolicyState(ecPolicy);
+ if (isDefault) {
+ assertEquals("The erasure coding policy should be of " +
+ "enabled state", ErasureCodingPolicyState.ENABLED, ecPolicyState);
+ } else {
+ assertEquals("The erasure coding policy should be of " +
+ "disabled state", ErasureCodingPolicyState.DISABLED, ecPolicyState);
+ }
+ assertFalse("Policy should be in disabled state in FSImage!",
+ isPolicyEnabledInFsImage(targetPolicy));
+
// Read file regardless of the erasure coding policy state
DFSTestUtil.readFileAsBytes(fs, filePath);
@@ -972,4 +989,15 @@ public class TestFSImage {
DFSTestUtil.readFileAsBytes(fs, filePath);
fs.delete(dirPath, true);
}
+
+ private boolean isPolicyEnabledInFsImage(ErasureCodingPolicy testPolicy) {
+ ErasureCodingPolicyInfo[] persistedPolicies =
+ ErasureCodingPolicyManager.getInstance().getPersistedPolicies();
+ for (ErasureCodingPolicyInfo p : persistedPolicies) {
+ if(p.getPolicy().getName().equals(testPolicy.getName())) {
+ return p.isEnabled();
+ }
+ }
+ throw new AssertionError("Policy is not found!");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
index a1946c2..0995f13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
@@ -463,7 +463,7 @@ public class TestNamenodeRetryCache {
assertTrue(namesystem.hasRetryCache());
cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) namesystem
.getRetryCache().getCacheSet();
- assertEquals("Retry cache size is wrong", 38, cacheSet.size());
+ assertEquals("Retry cache size is wrong", 39, cacheSet.size());
iter = cacheSet.iterator();
while (iter.hasNext()) {
CacheEntry entry = iter.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index 59afd8e..8ecf3a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -90,7 +90,7 @@ public class TestStripedINodeFile {
public ExpectedException thrown = ExpectedException.none();
@Before
- public void init() {
+ public void init() throws IOException {
Configuration conf = new HdfsConfiguration();
ErasureCodingPolicyManager.getInstance().init(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index bbcbaaa..3014778 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -195,7 +195,7 @@ public class TestRetryCacheWithHA {
FSNamesystem fsn1 = cluster.getNamesystem(1);
cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) fsn1
.getRetryCache().getCacheSet();
- assertEquals("Retry cache size is wrong", 38, cacheSet.size());
+ assertEquals("Retry cache size is wrong", 39, cacheSet.size());
iter = cacheSet.iterator();
while (iter.hasNext()) {
CacheEntry entry = iter.next();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org