You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aj...@apache.org on 2019/06/25 20:53:53 UTC
[hadoop] branch branch-2.8 updated: HADOOP-16350. Ability to tell
HDFS client not to request KMS Information from NameNode. Contributed by
Greg Senia, Ajay Kumar.
This is an automated email from the ASF dual-hosted git repository.
ajay pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.8 by this push:
new acfe2f9 HADOOP-16350. Ability to tell HDFS client not to request KMS Information from NameNode. Contributed by Greg Senia, Ajay Kumar.
acfe2f9 is described below
commit acfe2f9589fb95597446e772574cbffc5ab64e74
Author: Ajay Kumar <aj...@apache.org>
AuthorDate: Tue Jun 25 13:51:26 2019 -0700
HADOOP-16350. Ability to tell HDFS client not to request KMS Information from NameNode. Contributed by Greg Senia, Ajay Kumar.
---
.../org/apache/hadoop/fs/CommonConfigurationKeys.java | 14 ++++++++++++++
.../hadoop-common/src/main/resources/core-default.xml | 13 +++++++++++++
.../main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java | 19 ++++++++++++-------
.../org/apache/hadoop/hdfs/TestEncryptionZones.java | 11 +++++++++++
4 files changed, 50 insertions(+), 7 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index d8b8538..c06d036 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -345,4 +345,18 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
// HDFS client HTrace configuration.
public static final String FS_CLIENT_HTRACE_PREFIX = "fs.client.htrace.";
+
+ /*
+ * Ignore KMS default URI returned from NameNode.
+ * When set to true, kms uri is searched in the following order:
+ * 1. If there is a mapping in Credential's secrets map for namenode uri.
+ * 2. Fallback to local conf.
+ * If client choose to ignore KMS uri provided by NameNode then client
+ * should set KMS URI using 'hadoop.security.key.provider.path' to access
+ * the right KMS for encrypted files.
+ * */
+ public static final String DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI =
+ "dfs.client.ignore.namenode.default.kms.uri";
+ public static final boolean
+ DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT = false;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 2381109..f19749e 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2540,5 +2540,18 @@
in audit logs.
</description>
</property>
+ <property>
+ <name>dfs.client.ignore.namenode.default.kms.uri</name>
+ <value>false</value>
+ <description>
+ Ignore KMS default URI returned from NameNode.
+ When set to true, kms uri is searched in the following order:
+ 1. If there is a mapping in Credential's secrets map for namenode uri.
+ 2. Fallback to local conf. (i.e hadoop.security.key.provider.path)
+ If client choose to ignore KMS uri provided by NameNode then client
+ should set KMS URI using 'hadoop.security.key.provider.path' to access
+ the right KMS for encrypted files.
+ </description>
+ </property>
</configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
index fdef8f8..fafaf1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX;
import java.io.IOException;
@@ -163,19 +165,22 @@ public final class HdfsKMSUtil {
Credentials credentials = ugi.getCredentials();
byte[] keyProviderUriBytes =
credentials.getSecretKey(getKeyProviderMapKey(namenodeUri));
- if(keyProviderUriBytes != null) {
+ if (keyProviderUriBytes != null) {
keyProviderUri =
URI.create(DFSUtilClient.bytes2String(keyProviderUriBytes));
return keyProviderUri;
}
-
- if (keyProviderUriStr != null) {
- if (!keyProviderUriStr.isEmpty()) {
- keyProviderUri = URI.create(keyProviderUriStr);
+ if (keyProviderUri == null) {
+ // Check if NN provided uri is not null and ignore property is false.
+ if (keyProviderUriStr != null && !conf.getBoolean(
+ DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI,
+ DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT)) {
+ if (!keyProviderUriStr.isEmpty()) {
+ keyProviderUri = URI.create(keyProviderUriStr);
+ return keyProviderUri;
+ }
}
- return keyProviderUri;
}
-
// Last thing is to trust its own conf to be backwards compatible.
String keyProviderUriFromConf = conf.getTrimmed(
CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index e6497d3..d8985c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -110,6 +110,7 @@ import org.junit.Test;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
@@ -1827,6 +1828,16 @@ public class TestEncryptionZones {
Assert.assertEquals("Key Provider for client and namenode are different",
namenodeKeyProviderUri, cluster.getFileSystem().getClient()
.getKeyProviderUri());
+
+ // Ignore the key provider from NN.
+ clusterConf.setBoolean(
+ DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI, true);
+ Assert.assertEquals("Expecting Key Provider for client config",
+ "dummy://foo:bar@test_provider1", cluster.getFileSystem().getClient()
+ .getKeyProviderUri().toString());
+ Assert.assertNotEquals("Key Provider for client and namenode is different",
+ namenodeKeyProviderUri, cluster.getFileSystem().getClient()
+ .getKeyProviderUri().toString());
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org