You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2018/09/10 03:43:00 UTC
hadoop git commit: HDFS-13806. EC: No error message for unsetting EC
policy of the directory inherits the erasure coding policy from an ancestor
directory. Contributed by Ayush Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk de85f841d -> 30eceec34
HDFS-13806. EC: No error message for unsetting EC policy of the directory inherits the erasure coding policy from an ancestor directory. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30eceec3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30eceec3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30eceec3
Branch: refs/heads/trunk
Commit: 30eceec3420fc6be00d3878ba787bd9518d3ca0e
Parents: de85f84
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Sep 10 09:10:51 2018 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Sep 10 09:10:51 2018 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
.../hdfs/protocol/NoECPolicySetException.java | 37 ++++++++++++++++++++
.../server/namenode/FSDirErasureCodingOp.java | 4 +++
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 7 ++++
.../TestUnsetAndChangeDirectoryEcPolicy.java | 23 +++++++++---
.../test/resources/testErasureCodingConf.xml | 24 +++++++++++++
6 files changed, 92 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 61568d5..f4d11b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -131,6 +131,7 @@ import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.NoECPolicySetException;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
@@ -2750,7 +2751,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
throw re.unwrapRemoteException(AccessControlException.class,
SafeModeException.class,
UnresolvedPathException.class,
- FileNotFoundException.class);
+ FileNotFoundException.class, NoECPolicySetException.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java
new file mode 100644
index 0000000..de3054a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ *Thrown when no EC policy is set explicitly on the directory.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class NoECPolicySetException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public NoECPolicySetException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 2ba840e..b0bc5e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.protocol.NoECPolicySetException;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableUtils;
@@ -206,6 +207,9 @@ final class FSDirErasureCodingOp {
}
if (xAttrs != null) {
fsn.getEditLog().logRemoveXAttrs(src, xAttrs, logRetryCache);
+ } else {
+ throw new NoECPolicySetException(
+ "No erasure coding policy explicitly set on " + src);
}
return fsd.getAuditFileInfo(iip);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 56d453b..903a1e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.protocol.NoECPolicySetException;
import org.apache.hadoop.hdfs.util.ECPolicyLoader;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.tools.TableListing;
@@ -424,6 +425,12 @@ public class ECAdmin extends Configured implements Tool {
"non-empty directory will not automatically convert existing" +
" files to replicated data.");
}
+ } catch (NoECPolicySetException e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ System.err.println("Use '-setPolicy -path <PATH> -replicate' to enforce"
+ + " default replication policy irrespective of EC policy"
+ + " defined on parent.");
+ return 2;
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
return 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index aa14694..4f33ce7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.NoECPolicySetException;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
@@ -98,7 +99,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
fs.mkdirs(dirPath);
// Test unset a directory which has no EC policy
- fs.unsetErasureCodingPolicy(dirPath);
+ try {
+ fs.unsetErasureCodingPolicy(dirPath);
+ fail();
+ } catch (NoECPolicySetException e) {
+ }
// Set EC policy on directory
fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
@@ -126,8 +131,8 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
}
/*
- * Test nested directory with different EC policy.
- */
+ * Test nested directory with different EC policy.
+ */
@Test
public void testNestedEcPolicy() throws Exception {
final int numBlocks = 1;
@@ -199,7 +204,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
final Path replicateFilePath = new Path(rootPath, "rep_file");
// Test unset root path which has no EC policy
- fs.unsetErasureCodingPolicy(rootPath);
+ try {
+ fs.unsetErasureCodingPolicy(rootPath);
+ fail();
+ } catch (NoECPolicySetException e) {
+ }
// Set EC policy on root path
fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
@@ -238,7 +247,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
final ErasureCodingPolicy ec32Policy = SystemErasureCodingPolicies
.getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID);
- fs.unsetErasureCodingPolicy(rootPath);
+ try {
+ fs.unsetErasureCodingPolicy(rootPath);
+ fail();
+ } catch (NoECPolicySetException e) {
+ }
fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
// Create RS(6,3) EC policy file
DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index b47d50f..34f5176 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -360,6 +360,30 @@
</test>
<test>
+ <description>unsetPolicy : unset on non EC directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /ecdir</command>
+ <command>-fs NAMENODE -mkdir /ecdir/child</command>
+ <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /ecdir/child/ecfile</command>
+ <command>-fs NAMENODE -rmdir /ecdir/child</command>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>NoECPolicySetException: No erasure coding policy explicitly set on /ecdir/child</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Use '-setPolicy -path <PATH> -replicate' to enforce default replication policy irrespective of EC policy defined on parent.</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
<description>unsetPolicy : unset policy on non-empty directory</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org