You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/08/17 05:17:11 UTC
hadoop git commit: HDFS-11082. Provide replicated EC policy to
replicate files. Contributed by SammiChen.
Repository: hadoop
Updated Branches:
refs/heads/trunk 08aaa4b36 -> 96b3a6b97
HDFS-11082. Provide replicated EC policy to replicate files. Contributed by SammiChen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96b3a6b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96b3a6b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96b3a6b9
Branch: refs/heads/trunk
Commit: 96b3a6b9721e922d33fadc2459b561a85dbf9b8e
Parents: 08aaa4b
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Aug 16 22:17:06 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Aug 16 22:17:06 2017 -0700
----------------------------------------------------------------------
.../io/erasurecode/ErasureCodeConstants.java | 8 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
.../hadoop/hdfs/DistributedFileSystem.java | 6 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 6 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 5 ++
.../protocol/SystemErasureCodingPolicies.java | 14 ++++
.../namenode/ErasureCodingPolicyManager.java | 13 ++-
.../server/namenode/FSDirErasureCodingOp.java | 13 ++-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 2 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 24 +++++-
.../src/site/markdown/HDFSErasureCoding.md | 16 ++--
.../hadoop/hdfs/TestErasureCodingPolicies.java | 81 ++++++++++++++++++
.../hdfs/server/namenode/TestFSImage.java | 87 ++++++++++++++++++++
.../test/resources/testErasureCodingConf.xml | 78 +++++++++++++++++-
14 files changed, 331 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index e0d7946..d3c3b6b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -30,6 +30,7 @@ public final class ErasureCodeConstants {
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor";
public static final String HHXOR_CODEC_NAME = "hhxor";
+ public static final String REPLICATION_CODEC_NAME = "replication";
public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
RS_CODEC_NAME, 6, 3);
@@ -45,4 +46,11 @@ public final class ErasureCodeConstants {
public static final ECSchema RS_10_4_SCHEMA = new ECSchema(
RS_CODEC_NAME, 10, 4);
+
+ public static final ECSchema REPLICATION_1_2_SCHEMA = new ECSchema(
+ REPLICATION_CODEC_NAME, 1, 2);
+
+ public static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
+ public static final byte REPLICATION_POLICY_ID = (byte) 63;
+ public static final String REPLICATION_POLICY_NAME = REPLICATION_CODEC_NAME;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 969522d..47c14e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3044,7 +3044,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
*
* @param src path to get the information for
* @return Returns the policy information if file or directory on the path is
- * erasure coded, null otherwise
+ * erasure coded, null otherwise. Null will be returned if directory or file
+ * has REPLICATION policy.
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 8f82d03..ceec2b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2540,7 +2540,8 @@ public class DistributedFileSystem extends FileSystem {
*
* @param path The path of the file or directory
* @return Returns the policy information if file or directory on the path
- * is erasure coded, null otherwise
+ * is erasure coded, null otherwise. Null will be returned if directory or
+ * file has REPLICATION policy.
* @throws IOException
*/
public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
@@ -2567,7 +2568,8 @@ public class DistributedFileSystem extends FileSystem {
}
/**
- * Retrieve all the erasure coding policies supported by this file system.
+ * Retrieve all the erasure coding policies supported by this file system,
+ * excluding REPLICATION policy.
*
* @return all erasure coding policies supported by this file system.
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index eb9380d..b0e85e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1588,7 +1588,8 @@ public interface ClientProtocol {
/**
- * Get the erasure coding policies loaded in Namenode.
+ * Get the erasure coding policies loaded in Namenode, excluding REPLICATION
+ * policy.
*
* @throws IOException
*/
@@ -1604,7 +1605,8 @@ public interface ClientProtocol {
Map<String, String> getErasureCodingCodecs() throws IOException;
/**
- * Get the information about the EC policy for the path.
+ * Get the information about the EC policy for the path. Null will be returned
+ * if directory or file has REPLICATION policy.
*
* @param src path to get the info for
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
index 7afc377..501b67c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
@@ -25,6 +25,7 @@ import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
/**
* A policy about how to write/read/code an erasure coding file.
@@ -107,6 +108,10 @@ public final class ErasureCodingPolicy implements Serializable {
this.id = id;
}
+ public boolean isReplicationPolicy() {
+ return (id == ErasureCodeConstants.REPLICATION_POLICY_ID);
+ }
+
@Override
public boolean equals(Object o) {
if (o == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
index 2cd838b..f0efe76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
@@ -68,6 +68,13 @@ public final class SystemErasureCodingPolicies {
new ErasureCodingPolicy(ErasureCodeConstants.RS_10_4_SCHEMA,
DEFAULT_CELLSIZE, RS_10_4_POLICY_ID);
+ // REPLICATION policy is always enabled.
+ private static final ErasureCodingPolicy REPLICATION_POLICY =
+ new ErasureCodingPolicy(ErasureCodeConstants.REPLICATION_POLICY_NAME,
+ ErasureCodeConstants.REPLICATION_1_2_SCHEMA,
+ DEFAULT_CELLSIZE,
+ ErasureCodeConstants.REPLICATION_POLICY_ID);
+
private static final List<ErasureCodingPolicy> SYS_POLICIES =
Collections.unmodifiableList(Arrays.asList(
SYS_POLICY1, SYS_POLICY2, SYS_POLICY3, SYS_POLICY4,
@@ -118,4 +125,11 @@ public final class SystemErasureCodingPolicies {
public static ErasureCodingPolicy getByName(String name) {
return SYSTEM_POLICIES_BY_NAME.get(name);
}
+
+ /**
+ * Get the special REPLICATION policy.
+ */
+ public static ErasureCodingPolicy getReplicationPolicy() {
+ return REPLICATION_POLICY;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 18b8e8a..404a0aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +49,6 @@ public final class ErasureCodingPolicyManager {
public static Logger LOG = LoggerFactory.getLogger(
ErasureCodingPolicyManager.class);
- private static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
private int maxCellSize =
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT;
@@ -157,7 +157,13 @@ public final class ErasureCodingPolicyManager {
* Get enabled policy by policy name.
*/
public ErasureCodingPolicy getEnabledPolicyByName(String name) {
- return enabledPoliciesByName.get(name);
+ ErasureCodingPolicy ecPolicy = enabledPoliciesByName.get(name);
+ if (ecPolicy == null) {
+ if (name.equalsIgnoreCase(ErasureCodeConstants.REPLICATION_POLICY_NAME)) {
+ ecPolicy = SystemErasureCodingPolicies.getReplicationPolicy();
+ }
+ }
+ return ecPolicy;
}
/**
@@ -257,7 +263,8 @@ public final class ErasureCodingPolicyManager {
private byte getNextAvailablePolicyID() {
byte currentId = this.userPoliciesByID.keySet().stream()
- .max(Byte::compareTo).orElse(USER_DEFINED_POLICY_START_ID);
+ .max(Byte::compareTo).orElse(
+ ErasureCodeConstants.USER_DEFINED_POLICY_START_ID);
return (byte) (currentId + 1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 7895433..426b42b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -62,7 +62,7 @@ final class FSDirErasureCodingOp {
/**
* Check if the ecPolicyName is valid and enabled, return the corresponding
- * EC policy if is.
+ * EC policy if is, including the REPLICATION EC policy.
* @param fsn namespace
* @param ecPolicyName name of EC policy to be checked
* @return an erasure coding policy if ecPolicyName is valid and enabled
@@ -295,7 +295,12 @@ final class FSDirErasureCodingOp {
if (iip.getLastINode() == null) {
throw new FileNotFoundException("Path not found: " + iip.getPath());
}
- return getErasureCodingPolicyForPath(fsd, iip);
+
+ ErasureCodingPolicy ecPolicy = getErasureCodingPolicyForPath(fsd, iip);
+ if (ecPolicy != null && ecPolicy.isReplicationPolicy()) {
+ ecPolicy = null;
+ }
+ return ecPolicy;
}
/**
@@ -312,7 +317,8 @@ final class FSDirErasureCodingOp {
}
/**
- * Get the erasure coding policy. This does not do any permission checking.
+ * Get the erasure coding policy, including the REPLICATION policy. This does
+ * not do any permission checking.
*
* @param fsn namespace
* @param iip inodes in the path containing the file
@@ -350,6 +356,7 @@ final class FSDirErasureCodingOp {
return CodecRegistry.getInstance().getCodec2CoderCompactMap();
}
+ //return erasure coding policy for path, including REPLICATION policy
private static ErasureCodingPolicy getErasureCodingPolicyForPath(
FSDirectory fsd, INodesInPath iip) throws IOException {
Preconditions.checkNotNull(iip, "INodes cannot be null");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index a62cddd..7ab05d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -541,7 +541,7 @@ class FSDirWriteFileOp {
ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
fsd.getFSNamesystem(), existing);
}
- if (ecPolicy != null) {
+ if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) {
isStriped = true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 17a84f9..55d85ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.util.ECPolicyLoader;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.tools.TableListing;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
@@ -309,7 +310,8 @@ public class ECAdmin extends Configured implements Tool {
@Override
public String getShortUsage() {
- return "[" + getName() + " -path <path> -policy <policy>]\n";
+ return "[" + getName() +
+ " -path <path> [-policy <policy>] [-replicate]]\n";
}
@Override
@@ -318,9 +320,13 @@ public class ECAdmin extends Configured implements Tool {
listing.addRow("<path>", "The path of the file/directory to set " +
"the erasure coding policy");
listing.addRow("<policy>", "The name of the erasure coding policy");
+ listing.addRow("-replicate",
+ "force 3x replication scheme on the directory");
return getShortUsage() + "\n" +
"Set the erasure coding policy for a file/directory.\n\n" +
- listing.toString();
+ listing.toString() + "\n" +
+ "-replicate and -policy are optional arguments. They cannot been " +
+ "used at the same time";
}
@Override
@@ -332,14 +338,24 @@ public class ECAdmin extends Configured implements Tool {
return 1;
}
- final String ecPolicyName = StringUtils.popOptionWithArgument("-policy",
+ String ecPolicyName = StringUtils.popOptionWithArgument("-policy",
args);
+ final boolean replicate = StringUtils.popOption("-replicate", args);
if (args.size() > 0) {
System.err.println(getName() + ": Too many arguments");
return 1;
}
+ if (replicate) {
+ if (ecPolicyName != null) {
+ System.err.println(getName() +
+ ": -replicate and -policy cannot been used at the same time");
+ return 2;
+ }
+ ecPolicyName = ErasureCodeConstants.REPLICATION_POLICY_NAME;
+ }
+
final Path p = new Path(path);
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
try {
@@ -353,7 +369,7 @@ public class ECAdmin extends Configured implements Tool {
}
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
- return 2;
+ return 3;
}
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 4a48c2a..786b512 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -65,9 +65,11 @@ Architecture
2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work.
- Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, five built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, and `XOR-2-1-64k`.
+ Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, six built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, `XOR-2-1-64k` and `REPLICATION`.
- By default, all built-in erasure coding policies are disabled.
+ `REPLICATION` is a special policy. It can only be set on directory, to force the directory to adopt 3x replication scheme, instead of inheriting its ancestor's erasure coding policy. This policy makes it possible to interleave 3x replication scheme directory with erasure coding directory.
+
+ `REPLICATION` policy is always enabled. For other built-in policies, unless they are configured in `dfs.namenode.ec.policies.enabled` property, otherwise they are disabled by default.
Similar to HDFS storage policies, erasure coding policies are set on a directory. When a file is created, it inherits the EC policy of its nearest ancestor directory.
@@ -112,7 +114,7 @@ Deployment
what EC policies can be set by clients. It does not affect the behavior of already set file or directory-level EC policies.
By default, all built-in erasure coding policies are disabled. Typically, the cluster administrator will enable set of policies by including them
- in the `dfs .namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance,
+ in the `dfs.namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance,
for a cluster with 9 racks, a policy like `RS-10-4-64k` will not preserve rack-level fault-tolerance, and `RS-6-3-64k` or `RS-3-2-64k` might
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as
there are at least 14 DataNodes in the cluster.
@@ -153,7 +155,7 @@ Deployment
HDFS provides an `ec` subcommand to perform administrative commands related to erasure coding.
hdfs ec [generic options]
- [-setPolicy -policy <policyName> -path <path>]
+ [-setPolicy -path <path> [-policy <policyName>] [-replicate]]
[-getPolicy -path <path>]
[-unsetPolicy -path <path>]
[-listPolicies]
@@ -165,7 +167,7 @@ Deployment
Below are the details about each command.
- * `[-setPolicy -policy <policyName> -path <path>]`
+ * `[-setPolicy -path <path> [-policy <policyName>] [-replicate]]`
Sets an erasure coding policy on a directory at the specified path.
@@ -175,6 +177,10 @@ Below are the details about each command.
This parameter can be omitted if a 'dfs.namenode.ec.system.default.policy' configuration is set.
The EC policy of the path will be set with the default value in configuration.
+ `-replicate` apply the special `REPLICATION` policy on the directory, force the directory to adopt 3x replication scheme.
+
+ `-replicate` and `-policy <policyName>` are optional arguments. They cannot be specified at the same time.
+
* `[-getPolicy -path <path>]`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 22e118f..47cdf23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -732,4 +732,85 @@ public class TestErasureCodingPolicies {
}
});
}
+
+ @Test
+ public void testReplicationPolicy() throws Exception {
+ ErasureCodingPolicy replicaPolicy =
+ SystemErasureCodingPolicies.getReplicationPolicy();
+
+ final Path rootDir = new Path("/striped");
+ final Path replicaDir = new Path(rootDir, "replica");
+ final Path subReplicaDir = new Path(replicaDir, "replica");
+ final Path replicaFile = new Path(replicaDir, "file");
+ final Path subReplicaFile = new Path(subReplicaDir, "file");
+
+ fs.mkdirs(rootDir);
+ fs.setErasureCodingPolicy(rootDir, ecPolicy.getName());
+
+ // 1. At first, child directory will inherit parent's EC policy
+ fs.mkdirs(replicaDir);
+ fs.createFile(replicaFile).build().close();
+ HdfsFileStatus fileStatus = (HdfsFileStatus)fs.getFileStatus(replicaFile);
+ assertEquals("File should inherit EC policy.", ecPolicy, fileStatus
+ .getErasureCodingPolicy());
+ assertEquals("File should be a EC file.", true, fileStatus
+ .isErasureCoded());
+ assertEquals("File should have the same EC policy as its ancestor.",
+ ecPolicy, fs.getErasureCodingPolicy(replicaFile));
+ fs.delete(replicaFile, false);
+
+ // 2. Set replication policy on child directory, then get back the policy
+ fs.setErasureCodingPolicy(replicaDir, replicaPolicy.getName());
+ ErasureCodingPolicy temp = fs.getErasureCodingPolicy(replicaDir);
+ assertEquals("Directory should hide replication EC policy.",
+ null, temp);
+
+ // 3. New file will be replication file. Please be noted that replication
+ // policy only set on directory, not on file
+ fs.createFile(replicaFile).build().close();
+ assertEquals("Replication file should have default replication factor.",
+ fs.getDefaultReplication(),
+ fs.getFileStatus(replicaFile).getReplication());
+ fs.setReplication(replicaFile, (short) 2);
+ assertEquals("File should have replication factor as expected.",
+ 2, fs.getFileStatus(replicaFile).getReplication());
+ fileStatus = (HdfsFileStatus)fs.getFileStatus(replicaFile);
+ assertEquals("File should not have EC policy.", null, fileStatus
+ .getErasureCodingPolicy());
+ assertEquals("File should not be a EC file.", false,
+ fileStatus.isErasureCoded());
+ ErasureCodingPolicy ecPolicyOnFile = fs.getErasureCodingPolicy(replicaFile);
+ assertEquals("File should not have EC policy.", null, ecPolicyOnFile);
+ fs.delete(replicaFile, false);
+
+ // 4. New directory under replication directory, is also replication
+ // directory
+ fs.mkdirs(subReplicaDir);
+ assertEquals("Directory should inherit hiding replication EC policy.",
+ null, fs.getErasureCodingPolicy(subReplicaDir));
+ fs.createFile(subReplicaFile).build().close();
+ assertEquals("File should have default replication factor.",
+ fs.getDefaultReplication(),
+ fs.getFileStatus(subReplicaFile).getReplication());
+ fileStatus = (HdfsFileStatus)fs.getFileStatus(subReplicaFile);
+ assertEquals("File should not have EC policy.", null,
+ fileStatus.getErasureCodingPolicy());
+ assertEquals("File should not be a EC file.", false,
+ fileStatus.isErasureCoded());
+ assertEquals("File should not have EC policy.", null,
+ fs.getErasureCodingPolicy(subReplicaFile));
+ fs.delete(subReplicaFile, false);
+
+ // 5. Unset replication policy on directory, new file will be EC file
+ fs.unsetErasureCodingPolicy(replicaDir);
+ fs.createFile(subReplicaFile).build().close();
+ fileStatus = (HdfsFileStatus)fs.getFileStatus(subReplicaFile);
+ assertEquals("File should inherit EC policy.", ecPolicy,
+ fileStatus.getErasureCodingPolicy());
+ assertEquals("File should be a EC file.", true,
+ fileStatus.isErasureCoded());
+ assertEquals("File should have the same EC policy as its ancestor",
+ ecPolicy, fs.getErasureCodingPolicy(subReplicaFile));
+ fs.delete(subReplicaFile, false);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 22c40fb..9256056 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -723,4 +723,91 @@ public class TestFSImage {
.getBlockType());
assertEquals(defaultBlockType, BlockType.CONTIGUOUS);
}
+
+ /**
+ * Test if a INodeFile under a replication EC policy directory
+ * can be saved by FSImageSerialization and loaded by FSImageFormat#Loader.
+ */
+ @Test
+ public void testSaveAndLoadFileUnderReplicationPolicyDir()
+ throws IOException {
+ Configuration conf = new Configuration();
+ DFSTestUtil.enableAllECPolicies(conf);
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ FSNamesystem fsn = cluster.getNamesystem();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ ErasureCodingPolicy replicaPolicy =
+ SystemErasureCodingPolicies.getReplicationPolicy();
+ ErasureCodingPolicy defaultEcPolicy =
+ StripedFileTestUtil.getDefaultECPolicy();
+
+ final Path ecDir = new Path("/ec");
+ final Path replicaDir = new Path(ecDir, "replica");
+ final Path replicaFile1 = new Path(replicaDir, "f1");
+ final Path replicaFile2 = new Path(replicaDir, "f2");
+
+ // create root directory
+ fs.mkdir(ecDir, null);
+ fs.setErasureCodingPolicy(ecDir, defaultEcPolicy.getName());
+
+ // create directory, and set replication Policy
+ fs.mkdir(replicaDir, null);
+ fs.setErasureCodingPolicy(replicaDir, replicaPolicy.getName());
+
+ // create an empty file f1
+ fs.create(replicaFile1).close();
+
+ // create an under-construction file f2
+ FSDataOutputStream out = fs.create(replicaFile2, (short) 2);
+ out.writeBytes("hello");
+ ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet
+ .of(SyncFlag.UPDATE_LENGTH));
+
+ // checkpoint
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.restartNameNode();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+
+ assertTrue(fs.getFileStatus(ecDir).isDirectory());
+ assertTrue(fs.getFileStatus(replicaDir).isDirectory());
+ assertTrue(fs.exists(replicaFile1));
+ assertTrue(fs.exists(replicaFile2));
+
+ // check directories
+ assertEquals("Directory should have default EC policy.",
+ defaultEcPolicy, fs.getErasureCodingPolicy(ecDir));
+ assertEquals("Directory should hide replication EC policy.",
+ null, fs.getErasureCodingPolicy(replicaDir));
+
+ // check file1
+ assertEquals("File should not have EC policy.", null,
+ fs.getErasureCodingPolicy(replicaFile1));
+ // check internals of file2
+ INodeFile file2Node =
+ fsn.dir.getINode4Write(replicaFile2.toString()).asFile();
+ assertEquals("hello".length(), file2Node.computeFileSize());
+ assertTrue(file2Node.isUnderConstruction());
+ BlockInfo[] blks = file2Node.getBlocks();
+ assertEquals(1, blks.length);
+ assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
+ assertEquals("File should return expected replication factor.",
+ 2, blks[0].getReplication());
+ assertEquals("File should not have EC policy.", null,
+ fs.getErasureCodingPolicy(replicaFile2));
+ // check lease manager
+ Lease lease = fsn.leaseManager.getLease(file2Node);
+ Assert.assertNotNull(lease);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index c68c6d6..1baf355 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -101,7 +101,7 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>[-setPolicy -path <path> -policy <policy>]</expected-output>
+ <expected-output>[-setPolicy -path <path> [-policy <policy>] [-replicate]]</expected-output>
</comparator>
</comparators>
</test>
@@ -238,6 +238,29 @@
</test>
<test>
+ <description>setPolicy : set replication policy on a directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /ecdir</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
+ <command>-fs NAMENODE -mkdir /ecdir/replica</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -replicate -path /ecdir/replica</ec-admin-command>
+ <command>-fs NAMENODE -touchz /ecdir/replica/file</command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/replica/file</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /ecdir/replica/file</command>
+ <command>-fs NAMENODE -rmdir /ecdir/replica</command>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>is unspecified</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
<description>unsetPolicy : unset policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
@@ -453,7 +476,7 @@
<!-- Test illegal parameters -->
<test>
- <description>setPolicy : illegal parameters - path is missing</description>
+ <description>setPolicy : illegal parameters - path option is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy</ec-admin-command>
@@ -470,7 +493,7 @@
</test>
<test>
- <description>setPolicy : illegal parameters - policy name is missing</description>
+ <description>setPolicy : illegal parameters - path name is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path</ec-admin-command>
@@ -487,7 +510,7 @@
</test>
<test>
- <description>setPolicy : illegal parameters - too many arguments</description>
+ <description>setPolicy : illegal parameters - too many arguments case 1</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command>
@@ -504,6 +527,23 @@
</test>
<test>
+ <description>setPolicy : illegal parameters - too many arguments case 2</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /ecdir</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k -replicate /ecdir2</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-setPolicy: Too many arguments</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
<description>setPolicy : illegal parameters - invalidpolicy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
@@ -553,6 +593,36 @@
</test>
<test>
+ <description>setPolicy : illegal parameters - wrong spelling replicate </description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -replica</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-setPolicy: Too many arguments</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>setPolicy : illegal parameters - replicate and policy coexist</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k -replicate</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-replicate and -policy cannot been used at the same time</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
<description>setPolicy : set erasure coding policy without given a specific policy name</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org