You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/04/04 19:56:28 UTC
[04/50] [abbrv] hadoop git commit: HADOOP-14038. Rename ADLS
credential properties. Contributed by John Zhuge.
HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/253e3e78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/253e3e78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/253e3e78
Branch: refs/heads/HDFS-7240
Commit: 253e3e78abb9ebe0e8103d25538ff07875b33c84
Parents: 448ec81
Author: John Zhuge <jz...@cloudera.com>
Authored: Mon Feb 20 21:27:46 2017 -0800
Committer: John Zhuge <jz...@apache.org>
Committed: Mon Mar 27 23:05:57 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 15 +++-
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 38 +++++++--
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 +
.../src/site/markdown/index.md | 22 +++---
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 5 +-
.../fs/adl/TestValidateConfiguration.java | 83 ++++++++++++++++++--
6 files changed, 138 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index ada8b02..da4f6ac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -751,7 +751,20 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
this.loadDefaults = other.loadDefaults;
setQuietMode(other.getQuietMode());
}
-
+
+ /**
+ * Reload existing configuration instances.
+ */
+ public static synchronized void reloadExistingConfigurations() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reloading " + REGISTRY.keySet().size()
+ + " existing configurations");
+ }
+ for (Configuration conf : REGISTRY.keySet()) {
+ conf.reloadConfiguration();
+ }
+ }
+
/**
* Add a default resource. Resources are loaded in the order of the resources
* added.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index 8fc8e00..d3a5565 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.adl;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.DeprecationDelta;
/**
* Constants.
@@ -28,25 +30,25 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Evolving
public final class AdlConfKeys {
// OAuth2 Common Configuration
- public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2"
- + ".refresh.url";
+ public static final String AZURE_AD_REFRESH_URL_KEY =
+ "fs.adl.oauth2.refresh.url";
// optional when provider type is refresh or client id.
public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY =
- "dfs.adls.oauth2.access.token.provider";
+ "fs.adl.oauth2.access.token.provider";
public static final String AZURE_AD_CLIENT_ID_KEY =
- "dfs.adls.oauth2.client.id";
+ "fs.adl.oauth2.client.id";
public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
- "dfs.adls.oauth2.access.token.provider.type";
+ "fs.adl.oauth2.access.token.provider.type";
// OAuth Refresh Token Configuration
public static final String AZURE_AD_REFRESH_TOKEN_KEY =
- "dfs.adls.oauth2.refresh.token";
+ "fs.adl.oauth2.refresh.token";
public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken";
// OAuth Client Cred Token Configuration
public static final String AZURE_AD_CLIENT_SECRET_KEY =
- "dfs.adls.oauth2.credential";
+ "fs.adl.oauth2.credential";
public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED =
"ClientCredential";
@@ -75,7 +77,7 @@ public final class AdlConfKeys {
static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024;
static final String LATENCY_TRACKER_KEY =
- "adl.dfs.enable.client.latency.tracker";
+ "adl.enable.client.latency.tracker";
static final boolean LATENCY_TRACKER_DEFAULT = true;
static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY =
@@ -90,6 +92,26 @@ public final class AdlConfKeys {
"adl.feature.ownerandgroup.enableupn";
static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
+ public static void addDeprecatedKeys() {
+ Configuration.addDeprecations(new DeprecationDelta[]{
+ new DeprecationDelta("dfs.adls.oauth2.access.token.provider.type",
+ AZURE_AD_TOKEN_PROVIDER_TYPE_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.client.id",
+ AZURE_AD_CLIENT_ID_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.refresh.token",
+ AZURE_AD_REFRESH_TOKEN_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.refresh.url",
+ AZURE_AD_REFRESH_URL_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.credential",
+ AZURE_AD_CLIENT_SECRET_KEY),
+ new DeprecationDelta("dfs.adls.oauth2.access.token.provider",
+ AZURE_AD_TOKEN_PROVIDER_CLASS_KEY),
+ new DeprecationDelta("adl.dfs.enable.client.latency.tracker",
+ LATENCY_TRACKER_KEY)
+ });
+ Configuration.reloadExistingConfigurations();
+ }
+
private AdlConfKeys() {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index e0e273e..0b860b3 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -88,6 +88,10 @@ public class AdlFileSystem extends FileSystem {
private AccessTokenProvider tokenProvider;
private AzureADTokenProvider azureTokenProvider;
+ static {
+ AdlConfKeys.addDeprecatedKeys();
+ }
+
@Override
public String getScheme() {
return SCHEME;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index de8fda2..d2da858 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -83,7 +83,7 @@ Add the following properties to the cluster's `core-site.xml`
```xml
<property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
<value>RefreshToken</value>
</property>
```
@@ -95,12 +95,12 @@ service associated with the client id. See [*Active Directory Library For Java*]
```xml
<property>
- <name>dfs.adls.oauth2.client.id</name>
+ <name>fs.adl.oauth2.client.id</name>
<value></value>
</property>
<property>
- <name>dfs.adls.oauth2.refresh.token</name>
+ <name>fs.adl.oauth2.refresh.token</name>
<value></value>
</property>
```
@@ -133,22 +133,22 @@ Add the following properties to your `core-site.xml`
```xml
<property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
<value>ClientCredential</value>
</property>
<property>
- <name>dfs.adls.oauth2.refresh.url</name>
+ <name>fs.adl.oauth2.refresh.url</name>
<value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
</property>
<property>
- <name>dfs.adls.oauth2.client.id</name>
+ <name>fs.adl.oauth2.client.id</name>
<value>CLIENT ID FROM STEP 7 ABOVE</value>
</property>
<property>
- <name>dfs.adls.oauth2.credential</name>
+ <name>fs.adl.oauth2.credential</name>
<value>PASSWORD FROM STEP 7 ABOVE</value>
</property>
```
@@ -166,9 +166,9 @@ For additional reading on the credential provider API, see
##### Provisioning
```bash
-hadoop credential create dfs.adls.oauth2.client.id -value 123
+hadoop credential create fs.adl.oauth2.client.id -value 123
-provider localjceks://file/home/foo/adls.jceks
-hadoop credential create dfs.adls.oauth2.refresh.token -value 123
+hadoop credential create fs.adl.oauth2.refresh.token -value 123
-provider localjceks://file/home/foo/adls.jceks
```
@@ -176,7 +176,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
```xml
<property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
<value>RefreshToken</value>
</property>
<property>
@@ -190,7 +190,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
```bash
hadoop distcp
- [-D dfs.adls.oauth2.access.token.provider.type=RefreshToken
+ [-D fs.adl.oauth2.access.token.provider.type=RefreshToken
-D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
hdfs://<NameNode Hostname>:9001/user/foo/srcDir
adl://<Account Name>.azuredatalakestore.net/tgtDir/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 70f2a7f..3867e74 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -46,6 +46,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -121,8 +122,8 @@ public class TestAzureADTokenProvider {
Assert.fail("Initialization should have failed due no token provider "
+ "configuration");
} catch (IllegalArgumentException e) {
- Assert.assertTrue(
- e.getMessage().contains("dfs.adls.oauth2.access.token.provider"));
+ GenericTestUtils.assertExceptionContains(
+ AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, e);
}
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
CustomMockTokenProvider.class, AzureADTokenProvider.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
index 4cabaa3..3d51b42 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs.adl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -56,6 +58,11 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
.TOKEN_PROVIDER_TYPE_REFRESH_TOKEN;
import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
/**
* Validate configuration keys defined for adl storage file system instance.
*/
@@ -64,18 +71,18 @@ public class TestValidateConfiguration {
@Test
public void validateConfigurationKeys() {
Assert
- .assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
- Assert.assertEquals("dfs.adls.oauth2.access.token.provider",
+ .assertEquals("fs.adl.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
+ Assert.assertEquals("fs.adl.oauth2.access.token.provider",
AZURE_AD_TOKEN_PROVIDER_CLASS_KEY);
- Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
- Assert.assertEquals("dfs.adls.oauth2.refresh.token",
+ Assert.assertEquals("fs.adl.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
+ Assert.assertEquals("fs.adl.oauth2.refresh.token",
AZURE_AD_REFRESH_TOKEN_KEY);
Assert
- .assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
+ .assertEquals("fs.adl.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
Assert.assertEquals("adl.debug.override.localuserasfileowner",
ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER);
- Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type",
+ Assert.assertEquals("fs.adl.oauth2.access.token.provider.type",
AZURE_AD_TOKEN_PROVIDER_TYPE_KEY);
Assert.assertEquals("adl.feature.client.cache.readahead",
@@ -88,7 +95,7 @@ public class TestValidateConfiguration {
Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED);
- Assert.assertEquals("adl.dfs.enable.client.latency.tracker",
+ Assert.assertEquals("adl.enable.client.latency.tracker",
LATENCY_TRACKER_KEY);
Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT);
@@ -109,4 +116,66 @@ public class TestValidateConfiguration {
Assert.assertEquals(false,
ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
}
+
+ @Test
+ public void testSetDeprecatedKeys() throws ClassNotFoundException {
+ Configuration conf = new Configuration(true);
+ setDeprecatedKeys(conf);
+
+ // Force AdlFileSystem static initialization to register deprecated keys.
+ Class.forName(AdlFileSystem.class.getName());
+
+ assertDeprecatedKeys(conf);
+ }
+
+ @Test
+ public void testLoadDeprecatedKeys()
+ throws IOException, ClassNotFoundException {
+ Configuration saveConf = new Configuration(false);
+ setDeprecatedKeys(saveConf);
+
+ final File testRootDir = GenericTestUtils.getTestDir();
+ File confXml = new File(testRootDir, "testLoadDeprecatedKeys.xml");
+ OutputStream out = new FileOutputStream(confXml);
+ saveConf.writeXml(out);
+ out.close();
+
+ Configuration conf = new Configuration(true);
+ conf.addResource(confXml.toURI().toURL());
+
+ // Trigger loading the configuration resources by getting any key.
+ conf.get("dummy.key");
+
+ // Force AdlFileSystem static initialization to register deprecated keys.
+ Class.forName(AdlFileSystem.class.getName());
+
+ assertDeprecatedKeys(conf);
+ }
+
+ private void setDeprecatedKeys(Configuration conf) {
+ conf.set("dfs.adls.oauth2.access.token.provider.type", "dummyType");
+ conf.set("dfs.adls.oauth2.client.id", "dummyClientId");
+ conf.set("dfs.adls.oauth2.refresh.token", "dummyRefreshToken");
+ conf.set("dfs.adls.oauth2.refresh.url", "dummyRefreshUrl");
+ conf.set("dfs.adls.oauth2.credential", "dummyCredential");
+ conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass");
+ conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker");
+ }
+
+ private void assertDeprecatedKeys(Configuration conf) {
+ Assert.assertEquals("dummyType",
+ conf.get(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY));
+ Assert.assertEquals("dummyClientId",
+ conf.get(AZURE_AD_CLIENT_ID_KEY));
+ Assert.assertEquals("dummyRefreshToken",
+ conf.get(AZURE_AD_REFRESH_TOKEN_KEY));
+ Assert.assertEquals("dummyRefreshUrl",
+ conf.get(AZURE_AD_REFRESH_URL_KEY));
+ Assert.assertEquals("dummyCredential",
+ conf.get(AZURE_AD_CLIENT_SECRET_KEY));
+ Assert.assertEquals("dummyClass",
+ conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY));
+ Assert.assertEquals("dummyTracker",
+ conf.get(LATENCY_TRACKER_KEY));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org