You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2019/11/21 06:48:21 UTC
[hadoop] branch trunk updated: HDFS-14996. RBF: GetFileStatus fails
for directory with EC policy set in case of multiple destinations.
Contributed by Ayush Saxena.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
View the commit online:
https://github.com/apache/hadoop/commit/98d249dcdabb664ca82083a323afb1a8ed13c062
The following commit(s) were added to refs/heads/trunk by this push:
new 98d249d HDFS-14996. RBF: GetFileStatus fails for directory with EC policy set in case of multiple destinations. Contributed by Ayush Saxena.
98d249d is described below
commit 98d249dcdabb664ca82083a323afb1a8ed13c062
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Thu Nov 21 12:08:42 2019 +0530
HDFS-14996. RBF: GetFileStatus fails for directory with EC policy set in case of multiple destinations. Contributed by Ayush Saxena.
---
.../server/federation/router/FederationUtil.java | 9 ++++++-
...erRPCMultipleDestinationMountTableResolver.java | 8 ++++++
.../main/java/org/apache/hadoop/hdfs/DFSUtil.java | 30 ++++++++++++++++++++++
.../server/namenode/FSDirStatAndListingOp.java | 18 +++----------
4 files changed, 50 insertions(+), 15 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
index 0811531..df22c4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -24,9 +24,11 @@ import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
import java.net.URL;
import java.net.URLConnection;
+import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
@@ -249,6 +251,11 @@ public final class FederationUtil {
*/
public static HdfsFileStatus updateMountPointStatus(HdfsFileStatus dirStatus,
int children) {
+ // Get flags to set in new FileStatus.
+ EnumSet<HdfsFileStatus.Flags> flags =
+ DFSUtil.getFlags(dirStatus.isEncrypted(), dirStatus.isErasureCoded(),
+ dirStatus.isSnapshotEnabled(), dirStatus.hasAcl());
+ EnumSet.noneOf(HdfsFileStatus.Flags.class);
return new HdfsFileStatus.Builder().atime(dirStatus.getAccessTime())
.blocksize(dirStatus.getBlockSize()).children(children)
.ecPolicy(dirStatus.getErasureCodingPolicy())
@@ -258,6 +265,6 @@ public final class FederationUtil {
.owner(dirStatus.getOwner()).path(dirStatus.getLocalNameInBytes())
.perm(dirStatus.getPermission()).replication(dirStatus.getReplication())
.storagePolicy(dirStatus.getStoragePolicy())
- .symlink(dirStatus.getSymlinkInBytes()).build();
+ .symlink(dirStatus.getSymlinkInBytes()).flags(flags).build();
}
}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
index 72a243a..d00b93c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
@@ -410,6 +410,14 @@ public class TestRouterRPCMultipleDestinationMountTableResolver {
}
@Test
+ public void testECMultipleDestinations() throws Exception {
+ setupOrderMountPath(DestinationOrder.HASH_ALL);
+ Path mountPath = new Path("/mount/dir");
+ routerFs.setErasureCodingPolicy(mountPath, "RS-6-3-1024k");
+ assertTrue(routerFs.getFileStatus(mountPath).isErasureCoded());
+ }
+
+ @Test
public void testACLMultipleDestinations() throws Exception {
setupOrderMountPath(DestinationOrder.HASH_ALL);
Path mountPath = new Path("/mount/dir/dir");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index e198221..8ad11f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -53,6 +53,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Date;
+import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
@@ -83,6 +84,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Util;
@@ -1786,4 +1788,32 @@ public class DFSUtil {
}
}
+ /**
+ * Generates HdfsFileStatus flags.
+ * @param isEncrypted Sets HAS_CRYPT
+ * @param isErasureCoded Sets HAS_EC
+ * @param isSnapShottable Sets SNAPSHOT_ENABLED
+ * @param hasAcl Sets HAS_ACL
+ * @return HdfsFileStatus Flags
+ */
+ public static EnumSet<HdfsFileStatus.Flags> getFlags(
+ final boolean isEncrypted, final boolean isErasureCoded,
+ boolean isSnapShottable, boolean hasAcl) {
+ EnumSet<HdfsFileStatus.Flags> flags =
+ EnumSet.noneOf(HdfsFileStatus.Flags.class);
+ if (hasAcl) {
+ flags.add(HdfsFileStatus.Flags.HAS_ACL);
+ }
+ if (isEncrypted) {
+ flags.add(HdfsFileStatus.Flags.HAS_CRYPT);
+ }
+ if (isErasureCoded) {
+ flags.add(HdfsFileStatus.Flags.HAS_EC);
+ }
+ if (isSnapShottable) {
+ flags.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
+ }
+ return flags;
+ }
+
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index c7f43e5..7f07dd1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -449,22 +449,12 @@ class FSDirStatAndListingOp {
int childrenNum = node.isDirectory() ?
node.asDirectory().getChildrenNum(snapshot) : 0;
- EnumSet<HdfsFileStatus.Flags> flags =
- EnumSet.noneOf(HdfsFileStatus.Flags.class);
INodeAttributes nodeAttrs = fsd.getAttributes(iip);
boolean hasAcl = nodeAttrs.getAclFeature() != null;
- if (hasAcl) {
- flags.add(HdfsFileStatus.Flags.HAS_ACL);
- }
- if (isEncrypted) {
- flags.add(HdfsFileStatus.Flags.HAS_CRYPT);
- }
- if (isErasureCoded) {
- flags.add(HdfsFileStatus.Flags.HAS_EC);
- }
- if(isSnapShottable){
- flags.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
- }
+
+ EnumSet<HdfsFileStatus.Flags> flags =
+ DFSUtil.getFlags(isEncrypted, isErasureCoded, isSnapShottable, hasAcl);
+
return createFileStatus(
size,
node.isDirectory(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org