You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/03/03 02:45:24 UTC
[13/50] [abbrv] hadoop git commit: HDFS-11421. Make WebHDFS' ACLs
RegEx configurable. Contributed by Harsh J.
HDFS-11421. Make WebHDFS' ACLs RegEx configurable. Contributed by Harsh J.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e24ed47d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e24ed47d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e24ed47d
Branch: refs/heads/YARN-5734
Commit: e24ed47d9a19f34a4dd8d4bad9b5c78ca3dd1c2e
Parents: d2b3ba9
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Feb 24 16:49:16 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Feb 24 16:49:46 2017 -0800
----------------------------------------------------------------------
.../hdfs/client/HdfsClientConfigKeys.java | 2 ++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 6 +++-
.../hdfs/web/resources/AclPermissionParam.java | 17 +++++++++-
.../datanode/web/webhdfs/WebHdfsHandler.java | 10 ++++--
.../server/namenode/NameNodeHttpServer.java | 4 +++
.../src/main/resources/hdfs-default.xml | 8 +++++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 29 ++++++++++++++++-
.../hadoop/hdfs/web/resources/TestParam.java | 34 ++++++++++++++++++++
8 files changed, 104 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 7ad79e0..6f8c661 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -35,6 +35,8 @@ public interface HdfsClientConfigKeys {
String DFS_WEBHDFS_USER_PATTERN_KEY =
"dfs.webhdfs.user.provider.user.pattern";
String DFS_WEBHDFS_USER_PATTERN_DEFAULT = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$";
+ String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY =
+ "dfs.webhdfs.acl.provider.permission.pattern";
String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 135eef7..a9bc795 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -183,10 +183,14 @@ public class WebHdfsFileSystem extends FileSystem
) throws IOException {
super.initialize(uri, conf);
setConf(conf);
- /** set user pattern based on configuration file */
+
+ // set user and acl patterns based on configuration file
UserParam.setUserPattern(conf.get(
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+ AclPermissionParam.setAclPermissionPattern(conf.get(
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
boolean isOAuth = conf.getBoolean(
HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java
index 130c8fd..9ab3ad5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.permission.AclEntry;
/** AclPermission parameter. */
@@ -33,7 +34,7 @@ public class AclPermissionParam extends StringParam {
/** Default parameter value. */
public static final String DEFAULT = "";
- private static final Domain DOMAIN = new Domain(NAME,
+ private static Domain DOMAIN = new Domain(NAME,
Pattern.compile(DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
/**
@@ -49,6 +50,20 @@ public class AclPermissionParam extends StringParam {
super(DOMAIN,parseAclSpec(acl).equals(DEFAULT) ? null : parseAclSpec(acl));
}
+ @VisibleForTesting
+ public static Domain getAclPermissionPattern() {
+ return DOMAIN;
+ }
+
+ @VisibleForTesting
+ public static void setAclPermissionPattern(Domain dm) {
+ DOMAIN = dm;
+ }
+
+ public static void setAclPermissionPattern(String pattern) {
+ DOMAIN = new Domain(NAME, Pattern.compile(pattern));
+ }
+
@Override
public String getName() {
return NAME;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index f8c15fc..d2b2ec2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -48,11 +48,12 @@ import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
@@ -112,8 +113,11 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
this.confForCreate = confForCreate;
/** set user pattern based on configuration file */
UserParam.setUserPattern(
- conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
- DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+ conf.get(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+ AclPermissionParam.setAclPermissionPattern(
+ conf.get(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index a1959e4..e7e5f51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.AuthFilter;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.http.HttpConfig;
@@ -80,6 +81,9 @@ public class NameNodeHttpServer {
UserParam.setUserPattern(conf.get(
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+ AclPermissionParam.setAclPermissionPattern(conf.get(
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
// add authentication filter for webhdfs
final String className = conf.get(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 652b216..d23b967 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2478,6 +2478,14 @@
</property>
<property>
+ <name>dfs.webhdfs.acl.provider.permission.pattern</name>
+ <value>^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$</value>
+ <description>
+ Valid pattern for user and group names in webhdfs acl operations, it must be a valid java regex.
+ </description>
+</property>
+
+<property>
<name>dfs.webhdfs.socket.connect-timeout</name>
<value>60s</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 1ff04de..7cfaa99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -49,6 +49,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
+import com.google.common.collect.ImmutableList;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -64,6 +65,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -379,10 +383,17 @@ public class TestWebHDFS {
}
@Test(timeout=300000)
- public void testNumericalUserName() throws Exception {
+ public void testCustomizedUserAndGroupNames() throws Exception {
final Configuration conf = WebHdfsTestUtil.createConf();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ // Modify username pattern to allow numeric usernames
conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9" +
"._-]*[$]?$");
+ // Modify acl pattern to allow numeric and "@" characters user/groups in ACL spec
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ "^(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?" +
+ "(user|group|mask|other):[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?)*$");
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
try {
@@ -391,6 +402,7 @@ public class TestWebHDFS {
.setPermission(new Path("/"),
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
+ // Test a numeric username
UserGroupInformation.createUserForTesting("123", new String[]{"my-group"})
.doAs(new PrivilegedExceptionAction<Void>() {
@Override
@@ -399,6 +411,21 @@ public class TestWebHDFS {
WebHdfsConstants.WEBHDFS_SCHEME);
Path d = new Path("/my-dir");
Assert.assertTrue(fs.mkdirs(d));
+ // Test also specifying a default ACL with a numeric username
+ // and another of a groupname with '@'
+ fs.modifyAclEntries(d, ImmutableList.of(
+ new AclEntry.Builder()
+ .setPermission(FsAction.READ)
+ .setScope(AclEntryScope.DEFAULT)
+ .setType(AclEntryType.USER)
+ .setName("11010")
+ .build(),
+ new AclEntry.Builder()
+ .setPermission(FsAction.READ_WRITE)
+ .setType(AclEntryType.GROUP)
+ .setName("foo@bar")
+ .build()
+ ));
return null;
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24ed47d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
index 6449bf7..d444cb4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
@@ -351,6 +351,40 @@ public class TestParam {
LOG.info("EXPECTED: " + e);
}
}
+
+ @Test
+ public void testUserGroupOkAfterAlteringAclPattern() {
+ // Preserve default pattern value
+ AclPermissionParam.Domain oldDomain =
+ AclPermissionParam.getAclPermissionPattern();
+
+ // Override the pattern with one that accepts '@' and numbers
+ // in the first character of usernames/groupnames
+ String newPattern =
+ "^(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?" +
+ "(,(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
+
+ try {
+ AclPermissionParam.setAclPermissionPattern(newPattern);
+
+ String numericUserSpec = "user:110201:rwx";
+ AclPermissionParam aclNumericUserParam =
+ new AclPermissionParam(numericUserSpec);
+ Assert.assertEquals(numericUserSpec, aclNumericUserParam.getValue());
+
+ String oddGroupSpec = "group:foo@bar:rwx";
+ AclPermissionParam aclGroupWithDomainParam =
+ new AclPermissionParam(oddGroupSpec);
+ Assert.assertEquals(oddGroupSpec, aclGroupWithDomainParam.getValue());
+
+ } finally {
+ // Revert back to the default rules for remainder of tests
+ AclPermissionParam.setAclPermissionPattern(oldDomain);
+ }
+
+ }
@Test
public void testXAttrNameParam() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org