You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/04/17 15:15:34 UTC
[09/50] hadoop git commit: HADOOP-14174. Set default ADLS access
token provider type to ClientCredential. Contributed by John Zhuge.
HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed by John Zhuge.
Signed-off-by: John Zhuge <jz...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56e81f2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56e81f2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56e81f2a
Branch: refs/heads/trunk
Commit: 56e81f2a204d5e65d29222df1b2ba4d892f9c0d5
Parents: 96cbb4f
Author: John Zhuge <jz...@apache.org>
Authored: Tue Mar 28 09:40:14 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Apr 7 12:04:44 2017 -0700
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 63 ++++++++++++++++++++
.../conf/TestCommonConfigurationFields.java | 2 +-
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 2 +
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 3 +-
.../apache/hadoop/fs/adl/AdlMockWebServer.java | 3 +
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 3 +
.../hadoop/fs/adl/TestCustomTokenProvider.java | 4 ++
.../fs/adl/TestRelativePathFormation.java | 4 ++
8 files changed, 82 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b8f9904..521b013 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2456,6 +2456,7 @@
<!-- Azure Data Lake File System Configurations -->
+
<property>
<name>fs.adl.impl</name>
<value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
@@ -2465,6 +2466,68 @@
<name>fs.AbstractFileSystem.adl.impl</name>
<value>org.apache.hadoop.fs.adl.Adl</value>
</property>
+
+ <property>
+ <name>adl.feature.ownerandgroup.enableupn</name>
+ <value>false</value>
+ <description>
+ When true : User and Group in FileStatus/AclStatus response is
+ represented as user friendly name as per Azure AD profile.
+
+ When false (default) : User and Group in FileStatus/AclStatus
+ response is represented by the unique identifier from Azure AD
+ profile (Object ID as GUID).
+
+ For optimal performance, false is recommended.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
+ <value>ClientCredential</value>
+ <description>
+ Defines Azure Active Directory OAuth2 access token provider type.
+ Supported types are ClientCredential, RefreshToken, and Custom.
+ The ClientCredential type requires property fs.adl.oauth2.client.id,
+ fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
+ The RefreshToken type requires property fs.adl.oauth2.client.id and
+ fs.adl.oauth2.refresh.token.
+ The Custom type requires property fs.adl.oauth2.access.token.provider.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.client.id</name>
+ <value></value>
+ <description>The OAuth2 client id.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.credential</name>
+ <value></value>
+ <description>The OAuth2 access key.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.refresh.url</name>
+ <value></value>
+ <description>The OAuth2 token endpoint.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.refresh.token</name>
+ <value></value>
+ <description>The OAuth2 refresh token.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.access.token.provider</name>
+ <value></value>
+ <description>
+ The class name of the OAuth2 access token provider.
+ </description>
+ </property>
+
<!-- Azure Data Lake File System Configurations Ends Here-->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index cbfb6d1..8524973 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -105,7 +105,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
// ADL properties are in a different subtree
// - org.apache.hadoop.hdfs.web.ADLConfKeys
xmlPrefixToSkipCompare.add("adl.");
- xmlPropsToSkipCompare.add("fs.adl.impl");
+ xmlPrefixToSkipCompare.add("fs.adl.");
xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl");
// Azure properties are in a different class
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index d3a5565..31df222 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -40,6 +40,8 @@ public final class AdlConfKeys {
"fs.adl.oauth2.client.id";
public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
"fs.adl.oauth2.access.token.provider.type";
+ public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT =
+ TokenProviderType.ClientCredential;
// OAuth Refresh Token Configuration
public static final String AZURE_AD_REFRESH_TOKEN_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 0b860b3..e63f115 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -243,7 +243,8 @@ public class AdlFileSystem extends FileSystem {
Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
config, AdlFileSystem.class);
TokenProviderType type = conf.getEnum(
- AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
+ AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
+ AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT);
switch (type) {
case RefreshToken:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
index 55c8f81..d843d55 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
import com.squareup.okhttp.mockwebserver.MockWebServer;
@@ -84,6 +86,7 @@ public class AdlMockWebServer {
// Responses are returned in the same order that they are enqueued.
fs = new TestableAdlFileSystem();
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
CustomMockTokenProvider.class, AzureADTokenProvider.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 3867e74..36498c6 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -101,6 +101,7 @@ public class TestAzureADTokenProvider {
public void testCustomCredTokenProvider()
throws URISyntaxException, IOException {
Configuration conf = new Configuration();
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
CustomMockTokenProvider.class, AzureADTokenProvider.class);
@@ -115,6 +116,7 @@ public class TestAzureADTokenProvider {
public void testInvalidProviderConfigurationForType()
throws URISyntaxException, IOException {
Configuration conf = new Configuration();
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
URI uri = new URI("adl://localhost:8080");
AdlFileSystem fileSystem = new AdlFileSystem();
try {
@@ -136,6 +138,7 @@ public class TestAzureADTokenProvider {
Configuration conf = new Configuration();
URI uri = new URI("adl://localhost:8080");
AdlFileSystem fileSystem = new AdlFileSystem();
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
"wrong.classpath.CustomMockTokenProvider");
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
index c594c65..737534c 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
@@ -38,6 +38,8 @@ import java.util.Collection;
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
/**
* Test access token provider behaviour with custom token provider and for token
@@ -89,6 +91,8 @@ public class TestCustomTokenProvider extends AdlMockWebServer {
*/
public void init() throws IOException, URISyntaxException {
Configuration configuration = new Configuration();
+ configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
+ TokenProviderType.Custom);
configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
typeOfTokenProviderClass.getName());
fileSystems = new TestableAdlFileSystem[fsObjectCount];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
index 908f8b8..334c372 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
@@ -29,6 +29,8 @@ import java.net.URISyntaxException;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
/**
* This class verifies path conversion to SDK.
@@ -39,6 +41,8 @@ public class TestRelativePathFormation {
public void testToRelativePath() throws URISyntaxException, IOException {
AdlFileSystem fs = new AdlFileSystem();
Configuration configuration = new Configuration();
+ configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
+ TokenProviderType.Custom);
configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
"org.apache.hadoop.fs.adl.common.CustomMockTokenProvider");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org