You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/09/08 20:57:51 UTC
[19/45] hadoop git commit: HADOOP-14844. Remove requirement to
specify TenantGuid for MSI Token Provider. Contributed by Atul Sikaria.
HADOOP-14844. Remove requirement to specify TenantGuid for MSI Token Provider. Contributed by Atul Sikaria.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4661850
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4661850
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4661850
Branch: refs/heads/HDFS-10467
Commit: a4661850c1e0794baf493a468191e12681d68ab4
Parents: c35510a
Author: John Zhuge <jz...@apache.org>
Authored: Fri Sep 8 11:51:03 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Sep 8 11:51:03 2017 -0700
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 16 +++------------
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 -
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 +---
.../src/site/markdown/index.md | 21 ++++++--------------
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 4 ----
6 files changed, 11 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 23739b0..269a13c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2717,8 +2717,7 @@
fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
The RefreshToken type requires property fs.adl.oauth2.client.id and
fs.adl.oauth2.refresh.token.
- The MSI type requires properties fs.adl.oauth2.msi.port and
- fs.adl.oauth2.msi.tenantguid.
+ The MSI type reads optional property fs.adl.oauth2.msi.port, if specified.
The DeviceCode type requires property
fs.adl.oauth2.devicecode.clientapp.id.
The Custom type requires property fs.adl.oauth2.access.token.provider.
@@ -2762,17 +2761,8 @@
<value></value>
<description>
The localhost port for the MSI token service. This is the port specified
- when creating the Azure VM.
- Used by MSI token provider.
- </description>
- </property>
-
- <property>
- <name>fs.adl.oauth2.msi.tenantguid</name>
- <value></value>
- <description>
- The tenant guid for the Azure AAD tenant under which the azure data lake
- store account is created.
+ when creating the Azure VM. The default, if this setting is not specified,
+ is 50342.
Used by MSI token provider.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index b06c774..e32f9fc 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -110,7 +110,7 @@
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-data-lake-store-sdk</artifactId>
- <version>2.2.2</version>
+ <version>2.2.3</version>
</dependency>
<!-- ENDS HERE-->
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index f77d981..790902c 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -56,7 +56,6 @@ public final class AdlConfKeys {
// MSI Auth Configuration
public static final String MSI_PORT = "fs.adl.oauth2.msi.port";
- public static final String MSI_TENANT_GUID = "fs.adl.oauth2.msi.tenantguid";
// DeviceCode Auth configuration
public static final String DEVICE_CODE_CLIENT_APP_ID =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index a5e31e1..a496595 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -290,9 +290,7 @@ public class AdlFileSystem extends FileSystem {
private AccessTokenProvider getMsiBasedTokenProvider(
Configuration conf) throws IOException {
- int port = Integer.parseInt(getNonEmptyVal(conf, MSI_PORT));
- String tenantGuid = getPasswordString(conf, MSI_TENANT_GUID);
- return new MsiTokenProvider(port, tenantGuid);
+ return new MsiTokenProvider(conf.getInt(MSI_PORT, -1));
}
private AccessTokenProvider getDeviceCodeTokenProvider(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index e34da36..ca79321 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -164,15 +164,11 @@ Identity extension within the VM. The advantage of doing this is that the
credentials are managed by the extension, and do not have to be put into
core-site.xml.
-To use MSI, the following two steps are needed:
-1. Modify the VM deployment template to specify the port number of the token
- service exposed to localhost by the identity extension in the VM.
-2. Get your Azure ActiveDirectory Tenant ID:
- 1. Go to [the portal](https://portal.azure.com)
- 2. Under services in left nav, look for Azure Active Directory and click on it.
- 3. Click on Properties
- 4. Note down the GUID shown under "Directory ID" - this is your AAD tenant ID
-
+To use MSI, modify the VM deployment template to use the identity extension. Note the
+port number you specified in the template: this is the port number for the REST endpoint
+of the token service exposed to localhost by the identity extension in the VM. The default
+recommended port number is 50342 - if the recommended port number is used, then the msi.port
+setting below can be omitted in the configuration.
##### Configure core-site.xml
Add the following properties to your `core-site.xml`
@@ -185,12 +181,7 @@ Add the following properties to your `core-site.xml`
<property>
<name>fs.adl.oauth2.msi.port</name>
- <value>PORT NUMBER FROM STEP 1 ABOVE</value>
-</property>
-
-<property>
- <name>fs.adl.oauth2.msi.TenantGuid</name>
- <value>AAD TENANT ID GUID FROM STEP 2 ABOVE</value>
+ <value>PORT NUMBER FROM ABOVE (if different from the default of 50342)</value>
</property>
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 929b33a..12c2e3f 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -43,8 +43,6 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
import static org.apache.hadoop.fs.adl.AdlConfKeys.DEVICE_CODE_CLIENT_APP_ID;
-import static org.apache.hadoop.fs.adl.AdlConfKeys.MSI_PORT;
-import static org.apache.hadoop.fs.adl.AdlConfKeys.MSI_TENANT_GUID;
import static org.apache.hadoop.fs.adl.TokenProviderType.*;
import static org.junit.Assert.assertEquals;
@@ -107,8 +105,6 @@ public class TestAzureADTokenProvider {
throws IOException, URISyntaxException {
Configuration conf = new Configuration();
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, MSI);
- conf.set(MSI_PORT, "54321");
- conf.set(MSI_TENANT_GUID, "TENANT_GUID");
URI uri = new URI("adl://localhost:8080");
AdlFileSystem fileSystem = new AdlFileSystem();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org