You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/06/26 12:50:03 UTC
[29/50] hadoop git commit: HDFS-11606. Add CLI cmd to remove an
erasure code policy. Contributed by Tim Yao
HDFS-11606. Add CLI cmd to remove an erasure code policy. Contributed by Tim Yao
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5db3f984
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5db3f984
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5db3f984
Branch: refs/heads/HADOOP-13345
Commit: 5db3f9846882c51991d16853a5c431664f3f801f
Parents: a010b33
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Jun 21 14:01:55 2017 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Jun 21 14:01:55 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 6 +++
.../hadoop/hdfs/DistributedFileSystem.java | 11 ++++
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 29 ++++++++++-
.../hadoop/hdfs/protocol/ClientProtocol.java | 8 +++
.../ClientNamenodeProtocolTranslatorPB.java | 15 ++++++
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
.../src/main/proto/erasurecoding.proto | 7 +++
...tNamenodeProtocolServerSideTranslatorPB.java | 14 +++++
.../namenode/ErasureCodingPolicyManager.java | 25 +++++++++
.../server/namenode/FSDirErasureCodingOp.java | 13 +++++
.../hdfs/server/namenode/FSNamesystem.java | 22 ++++++++
.../hdfs/server/namenode/NameNodeRpcServer.java | 7 +++
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 54 ++++++++++++++++++--
.../src/site/markdown/HDFSErasureCoding.md | 4 ++
.../hadoop/hdfs/TestDistributedFileSystem.java | 29 +++++++++++
15 files changed, 242 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 51f04e0..d114f0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2777,6 +2777,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return namenode.addErasureCodingPolicies(policies);
}
+ public void removeErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ checkOpen();
+ namenode.removeErasureCodingPolicy(ecPolicyName);
+ }
+
public DFSInotifyEventInputStream getInotifyEventStream() throws IOException {
checkOpen();
return new DFSInotifyEventInputStream(namenode, tracer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 1a9ae48..3e09804 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2608,6 +2608,17 @@ public class DistributedFileSystem extends FileSystem {
}
/**
+ * Remove erasure coding policy.
+ *
+ * @param ecPolicyName The name of the policy to be removed.
+ * @throws IOException
+ */
+ public void removeErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ dfs.removeErasureCodingPolicy(ecPolicyName);
+ }
+
+ /**
* Unset the erasure coding policy from the source path.
*
* @param path The directory to unset the policy
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index 21de0ab..701bf0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -41,15 +41,16 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSInotifyEventInputStream;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
/**
* The public API for performing administrative functions on HDFS. Those writing
@@ -521,6 +522,32 @@ public class HdfsAdmin {
dfs.unsetErasureCodingPolicy(path);
}
+ /**
+ * Add Erasure coding policies to HDFS. For each policy input, schema and
+ * cellSize are musts, name and id are ignored. They will be automatically
+ * created and assigned by Namenode once the policy is successfully added, and
+ * will be returned in the response.
+ *
+ * @param policies The user defined ec policy list to add.
+ * @return Return the response list of adding operations.
+ * @throws IOException
+ */
+ public AddECPolicyResponse[] addErasureCodingPolicies(
+ ErasureCodingPolicy[] policies) throws IOException {
+ return dfs.addErasureCodingPolicies(policies);
+ }
+
+ /**
+ * Remove erasure coding policy.
+ *
+ * @param ecPolicyName The name of the policy to be removed.
+ * @throws IOException
+ */
+ public void removeErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ dfs.removeErasureCodingPolicy(ecPolicyName);
+ }
+
private void provisionEZTrash(Path path) throws IOException {
// make sure the path is an EZ
EncryptionZone ez = dfs.getEZForPath(path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index e132e04..313f322 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1563,6 +1563,14 @@ public interface ClientProtocol {
ErasureCodingPolicy[] policies) throws IOException;
/**
+ * Remove erasure coding policy.
+ * @param ecPolicyName The name of the policy to be removed.
+ * @throws IOException
+ */
+ @AtMostOnce
+ void removeErasureCodingPolicy(String ecPolicyName) throws IOException;
+
+ /**
* Get the erasure coding policies loaded in Namenode.
*
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 0d517f80..d913f81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -185,6 +185,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodin
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto;
@@ -1694,6 +1695,20 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
+ public void removeErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ RemoveErasureCodingPolicyRequestProto.Builder builder =
+ RemoveErasureCodingPolicyRequestProto.newBuilder();
+ builder.setEcPolicyName(ecPolicyName);
+ RemoveErasureCodingPolicyRequestProto req = builder.build();
+ try {
+ rpcProxy.removeErasureCodingPolicy(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
try {
GetErasureCodingPoliciesResponseProto response = rpcProxy
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index c56c0b1..d3e15d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -955,6 +955,8 @@ service ClientNamenodeProtocol {
returns(GetErasureCodingPoliciesResponseProto);
rpc addErasureCodingPolicies(AddErasureCodingPoliciesRequestProto)
returns(AddErasureCodingPoliciesResponseProto);
+ rpc removeErasureCodingPolicy(RemoveErasureCodingPolicyRequestProto)
+ returns(RemoveErasureCodingPolicyResponseProto);
rpc getErasureCodingPolicy(GetErasureCodingPolicyRequestProto)
returns(GetErasureCodingPolicyResponseProto);
rpc getErasureCodingCodecs(GetErasureCodingCodecsRequestProto)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
index ea878ec..ed0fd81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
@@ -61,6 +61,13 @@ message AddErasureCodingPoliciesResponseProto {
repeated AddECPolicyResponseProto responses = 1;
}
+message RemoveErasureCodingPolicyRequestProto {
+ required string ecPolicyName = 1;
+}
+
+message RemoveErasureCodingPolicyResponseProto {
+}
+
message UnsetErasureCodingPolicyRequestProto {
required string src = 1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 7135ff1..16adc70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -228,6 +228,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodin
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto;
@@ -1694,6 +1696,18 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+ public RemoveErasureCodingPolicyResponseProto removeErasureCodingPolicy(
+ RpcController controller, RemoveErasureCodingPolicyRequestProto request)
+ throws ServiceException {
+ try {
+ server.removeErasureCodingPolicy(request.getEcPolicyName());
+ return RemoveErasureCodingPolicyResponseProto.newBuilder().build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
public GetErasureCodingPolicyResponseProto getErasureCodingPolicy(RpcController controller,
GetErasureCodingPolicyRequestProto request) throws ServiceException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 0feb79c..d22e47e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;
@@ -68,6 +69,11 @@ public final class ErasureCodingPolicyManager {
private Map<Byte, ErasureCodingPolicy> userPoliciesByID;
/**
+ * All removed policies sorted by name.
+ */
+ private Map<String, ErasureCodingPolicy> removedPoliciesByName;
+
+ /**
* All enabled policies maintained in NN memory for fast querying,
* identified and sorted by its name.
*/
@@ -91,6 +97,7 @@ public final class ErasureCodingPolicyManager {
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT);
this.userPoliciesByID = new TreeMap<>();
this.userPoliciesByName = new TreeMap<>();
+ this.removedPoliciesByName = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
for (String policyName : policyNames) {
if (policyName.trim().isEmpty()) {
@@ -243,4 +250,22 @@ public final class ErasureCodingPolicyManager {
.max(Byte::compareTo).orElse(USER_DEFINED_POLICY_START_ID);
return (byte) (currentId + 1);
}
+
+ public synchronized void removePolicy(String name) {
+ if (SystemErasureCodingPolicies.getByName(name) != null) {
+ throw new IllegalArgumentException("System erasure coding policy " +
+ name + " cannot be removed");
+ }
+ ErasureCodingPolicy policy = userPoliciesByName.get(name);
+ if (policy == null) {
+ throw new IllegalArgumentException("The policy name " +
+ name + " does not exists");
+ }
+ enabledPoliciesByName.remove(name);
+ removedPoliciesByName.put(name, policy);
+ }
+
+ public List<ErasureCodingPolicy> getRemovedPolicies() {
+ return removedPoliciesByName.values().stream().collect(Collectors.toList());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index bedbe7d..a040262 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -218,6 +218,19 @@ final class FSDirErasureCodingOp {
return fsn.getErasureCodingPolicyManager().addPolicy(policy);
}
+ /**
+ * Remove an erasure coding policy.
+ *
+ * @param fsn namespace
+ * @param ecPolicyName the name of the policy to be removed
+ * @throws IOException
+ */
+ static void removeErasureCodePolicy(final FSNamesystem fsn,
+ String ecPolicyName) throws IOException {
+ Preconditions.checkNotNull(ecPolicyName);
+ fsn.getErasureCodingPolicyManager().removePolicy(ecPolicyName);
+ }
+
private static List<XAttr> removeErasureCodingPolicyXAttr(
final FSNamesystem fsn, final INodesInPath srcIIP) throws IOException {
FSDirectory fsd = fsn.getFSDirectory();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 3f7f1ca..f7acb55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7087,6 +7087,28 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * Remove an erasure coding policy.
+ * @param ecPolicyName the name of the policy to be removed
+ * @throws IOException
+ */
+ void removeErasureCodingPolicy(String ecPolicyName) throws IOException {
+ final String operationName = "removeErasureCodingPolicy";
+ checkOperation(OperationCategory.WRITE);
+ boolean success = false;
+ writeLock();
+ try {
+ FSDirErasureCodingOp.removeErasureCodePolicy(this, ecPolicyName);
+ success = true;
+ } finally {
+ writeUnlock(operationName);
+ if (success) {
+ getEditLog().logSync();
+ }
+ logAuditEvent(success, operationName, null, null, null);
+ }
+ }
+
+ /**
* Unset an erasure coding policy from the given path.
* @param srcArg The path of the target directory.
* @throws AccessControlException if the caller is not the superuser.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index e11a546..599178a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -2301,6 +2301,13 @@ public class NameNodeRpcServer implements NamenodeProtocols {
return namesystem.addECPolicies(policies);
}
+ @Override
+ public void removeErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ checkNNStartup();
+ namesystem.removeErasureCodingPolicy(ecPolicyName);
+ }
+
@Override // ReconfigurationProtocol
public void startReconfiguration() throws IOException {
checkNNStartup();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 2041a05..a983056 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -199,7 +199,7 @@ public class ECAdmin extends Configured implements Tool {
}
}
- /** Command to get the erasure coding policy for a file or directory */
+ /** Command to get the erasure coding policy for a file or directory. */
private static class GetECPolicyCommand implements AdminHelper.Command {
@Override
public String getName() {
@@ -254,7 +254,54 @@ public class ECAdmin extends Configured implements Tool {
}
}
- /** Command to set the erasure coding policy to a file/directory */
+ /** Command to remove an erasure coding policy. */
+ private static class RemoveECPolicyCommand implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-removePolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -policy <policy>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<policy>", "The name of the erasure coding policy");
+ return getShortUsage() + "\n" +
+ "Remove an erasure coding policy.\n" +
+ listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String ecPolicyName = StringUtils.popOptionWithArgument(
+ "-policy", args);
+ if (ecPolicyName == null) {
+ System.err.println("Please specify the policy name.\nUsage: " +
+ getLongUsage());
+ return 1;
+ }
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ try {
+ dfs.removeErasureCodingPolicy(ecPolicyName);
+ System.out.println("Erasure coding policy " + ecPolicyName +
+ "is removed");
+ } catch (IOException e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ /** Command to set the erasure coding policy to a file/directory. */
private static class SetECPolicyCommand implements AdminHelper.Command {
@Override
public String getName() {
@@ -313,7 +360,7 @@ public class ECAdmin extends Configured implements Tool {
}
}
- /** Command to unset the erasure coding policy set for a file/directory */
+ /** Command to unset the erasure coding policy set for a file/directory. */
private static class UnsetECPolicyCommand
implements AdminHelper.Command {
@@ -421,6 +468,7 @@ public class ECAdmin extends Configured implements Tool {
new ListECPoliciesCommand(),
new AddECPoliciesCommand(),
new GetECPolicyCommand(),
+ new RemoveECPolicyCommand(),
new SetECPolicyCommand(),
new UnsetECPolicyCommand(),
new ListECCodecsCommand()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 51f54a2..4d0b2e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -187,3 +187,7 @@ Below are the details about each command.
* `[-listCodecs]`
Get the list of supported erasure coding codecs and coders in system. A coder is an implementation of a codec. A codec can have different implementations, thus different coders. The coders for a codec are listed in a fall back order.
+
+* `[-removePolicy -policy <policyName>]`
+
+ Remove an erasure coding policy.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5db3f984/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 4479419..1d24f52 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
@@ -85,7 +86,9 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.net.DNSToSwitchMapping;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.ScriptBasedMapping;
@@ -1538,4 +1541,30 @@ public class TestDistributedFileSystem {
assertEquals(16 * 2, status.getLen());
}
}
+
+ @Test
+ public void testRemoveErasureCodingPolicy() throws Exception {
+ Configuration conf = getTestConfiguration();
+ MiniDFSCluster cluster = null;
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ ECSchema toAddSchema = new ECSchema("rs", 3, 2);
+ ErasureCodingPolicy toAddPolicy =
+ new ErasureCodingPolicy(toAddSchema, 128 * 1024, (byte) 254);
+ String policyName = toAddPolicy.getName();
+ ErasureCodingPolicy[] policies = new ErasureCodingPolicy[]{toAddPolicy};
+ fs.addErasureCodingPolicies(policies);
+ assertEquals(policyName, ErasureCodingPolicyManager.getInstance().
+ getByName(policyName).getName());
+ fs.removeErasureCodingPolicy(policyName);
+ assertEquals(policyName, ErasureCodingPolicyManager.getInstance().
+ getRemovedPolicies().get(0).getName());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org