You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/02/22 23:34:27 UTC
[41/50] [abbrv] hadoop git commit: HADOOP-14017. User friendly name
for ADLS user and group. Contributed by Vishwajeet Dusane
HADOOP-14017. User friendly name for ADLS user and group. Contributed by Vishwajeet Dusane
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/924def78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/924def78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/924def78
Branch: refs/heads/HADOOP-13345
Commit: 924def78544a64449785f305cb6984c3559aea4d
Parents: 2158496
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Feb 21 13:44:42 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Feb 21 13:44:42 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 4 +++
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 23 +++++++++++---
.../src/site/markdown/index.md | 26 +++++++++++++++
.../fs/adl/TestValidateConfiguration.java | 9 ++++++
.../apache/hadoop/fs/adl/live/TestMetadata.java | 33 ++++++++++++++++++++
5 files changed, 91 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/924def78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index 21120df..7d31103 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -87,6 +87,10 @@ public final class AdlConfKeys {
"adl.feature.support.acl.bit";
static final boolean ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION_DEFAULT = true;
+ static final String ADL_ENABLEUPN_FOR_OWNERGROUP_KEY =
+ "adl.feature.ownerandgroup.enableupn";
+ static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
+
private AdlConfKeys() {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/924def78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index fb0feda..e0e273e 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -32,6 +32,7 @@ import com.microsoft.azure.datalake.store.DirectoryEntry;
import com.microsoft.azure.datalake.store.DirectoryEntryType;
import com.microsoft.azure.datalake.store.IfExists;
import com.microsoft.azure.datalake.store.LatencyTracker;
+import com.microsoft.azure.datalake.store.UserGroupRepresentation;
import com.microsoft.azure.datalake.store.oauth2.AccessTokenProvider;
import com.microsoft.azure.datalake.store.oauth2.ClientCredsTokenProvider;
import com.microsoft.azure.datalake.store.oauth2.RefreshTokenBasedTokenProvider;
@@ -80,6 +81,8 @@ public class AdlFileSystem extends FileSystem {
private ADLStoreClient adlClient;
private Path workingDirectory;
private boolean aclBitStatus;
+ private UserGroupRepresentation oidOrUpn;
+
// retained for tests
private AccessTokenProvider tokenProvider;
@@ -181,6 +184,11 @@ public class AdlFileSystem extends FileSystem {
if (!trackLatency) {
LatencyTracker.disable();
}
+
+ boolean enableUPN = conf.getBoolean(ADL_ENABLEUPN_FOR_OWNERGROUP_KEY,
+ ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
+ oidOrUpn = enableUPN ? UserGroupRepresentation.UPN :
+ UserGroupRepresentation.OID;
}
/**
@@ -439,7 +447,8 @@ public class AdlFileSystem extends FileSystem {
@Override
public FileStatus getFileStatus(final Path f) throws IOException {
statistics.incrementReadOps(1);
- DirectoryEntry entry = adlClient.getDirectoryEntry(toRelativeFilePath(f));
+ DirectoryEntry entry =
+ adlClient.getDirectoryEntry(toRelativeFilePath(f), oidOrUpn);
return toFileStatus(entry, f);
}
@@ -456,7 +465,7 @@ public class AdlFileSystem extends FileSystem {
public FileStatus[] listStatus(final Path f) throws IOException {
statistics.incrementReadOps(1);
List<DirectoryEntry> entries =
- adlClient.enumerateDirectory(toRelativeFilePath(f));
+ adlClient.enumerateDirectory(toRelativeFilePath(f), oidOrUpn);
return toFileStatuses(entries, f);
}
@@ -749,8 +758,8 @@ public class AdlFileSystem extends FileSystem {
@Override
public AclStatus getAclStatus(final Path path) throws IOException {
statistics.incrementReadOps(1);
- com.microsoft.azure.datalake.store.acl.AclStatus adlStatus = adlClient
- .getAclStatus(toRelativeFilePath(path));
+ com.microsoft.azure.datalake.store.acl.AclStatus adlStatus =
+ adlClient.getAclStatus(toRelativeFilePath(path), oidOrUpn);
AclStatus.Builder aclStatusBuilder = new AclStatus.Builder();
aclStatusBuilder.owner(adlStatus.owner);
aclStatusBuilder.group(adlStatus.group);
@@ -963,4 +972,10 @@ public class AdlFileSystem extends FileSystem {
}
return new String(passchars);
}
+
+ @VisibleForTesting
+ public void setUserGroupRepresentationAsUPN(boolean enableUPN) {
+ oidOrUpn = enableUPN ? UserGroupRepresentation.UPN :
+ UserGroupRepresentation.OID;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/924def78/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index 5037db6..6d9e173 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -26,6 +26,7 @@
* [Protecting the Credentials with Credential Providers](#Credential_Provider)
* [Enabling ADL Filesystem](#Enabling_ADL)
* [Accessing adl URLs](#Accessing_adl_URLs)
+ * [User/Group Representation](#OIDtoUPNConfiguration)
* [Testing the hadoop-azure Module](#Testing_the_hadoop-azure_Module)
## <a name="Introduction" />Introduction
@@ -42,6 +43,8 @@ The jar file is named azure-datalake-store.jar.
* Can act as a source of data in a MapReduce job, or a sink.
* Tested on both Linux and Windows.
* Tested for scale.
+* API setOwner/setAcl/removeAclEntries/modifyAclEntries accepts UPN or OID
+ (Object ID) as user and group name.
## <a name="Limitations" />Limitations
Partial or no support for the following operations :
@@ -221,6 +224,29 @@ commands demonstrate access to a storage account named `youraccount`.
> hadoop fs -cat adl://yourcontainer.azuredatalakestore.net/testDir/testFile
test file content
+
+### <a name="OIDtoUPNConfiguration" />User/Group Representation
+The hadoop-azure-datalake module provides support for configuring how
+User/Group information is represented during
+getFileStatus/listStatus/getAclStatus.
+
+Add the following properties to your core-site.xml
+
+ <property>
+ <name>adl.feature.ownerandgroup.enableupn</name>
+ <value>true</value>
+ <description>
+ When true : User and Group in FileStatus/AclStatus response is
+ represented as user friendly name as per Azure AD profile.
+
+ When false (default) : User and Group in FileStatus/AclStatus
+ response is represented by the unique identifier from Azure AD
+ profile (Object ID as GUID).
+
+ For performance optimization, Recommended default value.
+ </description>
+ </property>
+
## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/924def78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
index e3025b2..4cabaa3 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
@@ -27,6 +27,10 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
import static org.apache.hadoop.fs.adl.AdlConfKeys
.ADL_DEBUG_SET_LOCAL_USER_AS_OWNER_DEFAULT;
import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_ENABLEUPN_FOR_OWNERGROUP_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
.ADL_EXPERIMENT_POSITIONAL_READ_DEFAULT;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.ADL_EXPERIMENT_POSITIONAL_READ_KEY;
@@ -99,5 +103,10 @@ public class TestValidateConfiguration {
Assert.assertEquals(false, ADL_DEBUG_SET_LOCAL_USER_AS_OWNER_DEFAULT);
Assert.assertEquals(4 * 1024 * 1024, DEFAULT_READ_AHEAD_BUFFER_SIZE);
Assert.assertEquals(4 * 1024 * 1024, DEFAULT_WRITE_AHEAD_BUFFER_SIZE);
+
+ Assert.assertEquals("adl.feature.ownerandgroup.enableupn",
+ ADL_ENABLEUPN_FOR_OWNERGROUP_KEY);
+ Assert.assertEquals(false,
+ ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/924def78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
index 3b9e7da..dbcaa39 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
@@ -32,6 +33,8 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.UUID;
+import static org.junit.Assert.fail;
+
/**
* This class is responsible for testing ContentSummary, ListStatus on
* file/folder.
@@ -107,5 +110,35 @@ public class TestMetadata {
.assertEquals(path.makeQualified(fs.getUri(), fs.getWorkingDirectory()),
statuses[0].getPath());
}
+
+ @Test
+ public void testUserRepresentationConfiguration() throws IOException {
+ // Validating actual user/group OID or friendly name is outside scope of
+ // this test.
+ Path path = new Path(parent, "a.txt");
+ AdlFileSystem fs = (AdlFileSystem) adlStore;
+
+ // When set to true, User/Group information should be user friendly name.
+ // That is non GUID value.
+ fs.setUserGroupRepresentationAsUPN(false);
+ fs.createNewFile(path);
+ Assert.assertTrue(fs.isFile(path));
+ FileStatus fileStatus = fs.getFileStatus(path);
+ UUID.fromString(fileStatus.getGroup());
+ UUID.fromString(fileStatus.getOwner());
+
+ // When set to false, User/Group information should be AAD represented
+ // unique OID. That is GUID value.
+ // Majority of the cases, user friendly name would not be GUID value.
+ fs.setUserGroupRepresentationAsUPN(true);
+ fileStatus = fs.getFileStatus(path);
+ try {
+ UUID.fromString(fileStatus.getGroup());
+ UUID.fromString(fileStatus.getOwner());
+ fail("Expected user friendly name to be non guid value.");
+ } catch (IllegalArgumentException e) {
+ // expected to fail since
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org