You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/04/21 21:36:04 UTC
[20/37] hadoop git commit: HADOOP-14241. Add ADLS sensitive config
keys to default list. Contributed by John Zhuge.
HADOOP-14241. Add ADLS sensitive config keys to default list. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0344bea3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0344bea3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0344bea3
Branch: refs/heads/HDFS-9806
Commit: 0344bea3fd4031622edd828a610c9fdc23c53d26
Parents: dd43b89
Author: John Zhuge <jz...@apache.org>
Authored: Sat Mar 25 11:59:53 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Wed Apr 19 11:50:54 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/ConfigRedactor.java | 3 ++-
.../hadoop/fs/CommonConfigurationKeysPublic.java | 16 +++++++++-------
.../src/main/resources/core-default.xml | 15 ++++++++++++---
.../org/apache/hadoop/conf/TestConfigRedactor.java | 17 ++++++++++++++++-
4 files changed, 39 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344bea3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
index 0ba756c..90b260e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
@@ -42,7 +42,8 @@ public class ConfigRedactor {
String sensitiveRegexList = conf.get(
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS,
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT);
- List<String> sensitiveRegexes = Arrays.asList(sensitiveRegexList.split(","));
+ List<String> sensitiveRegexes =
+ Arrays.asList(sensitiveRegexList.trim().split("[,\\s]+"));
compiledPatterns = new ArrayList<Pattern>();
for (String regex : sensitiveRegexes) {
Pattern p = Pattern.compile(regex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344bea3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 6b267dc..5f34b7b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -810,12 +810,14 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
"hadoop.security.sensitive-config-keys";
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
- "secret$" + "," +
- "password$" + "," +
- "ssl.keystore.pass$" + "," +
- "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
- "fs.azure\\.account.key.*" + "," +
- "dfs.webhdfs.oauth2.[a-z]+.token" + "," +
- HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
+ String.join(",",
+ "secret$",
+ "password$",
+ "ssl.keystore.pass$",
+ "fs.s3.*[Ss]ecret.?[Kk]ey",
+ "fs.azure\\.account.key.*",
+ "credential$",
+ "oauth.*token$",
+ HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344bea3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 4f37c65..283588a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -517,9 +517,18 @@
<property>
<name>hadoop.security.sensitive-config-keys</name>
- <value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
- <description>A comma-separated list of regular expressions to match against
- configuration keys that should be redacted where appropriate, for
+ <value>
+ secret$
+ password$
+ ssl.keystore.pass$
+ fs.s3.*[Ss]ecret.?[Kk]ey
+ fs.azure.account.key.*
+ credential$
+ oauth.*token$
+ hadoop.security.sensitive-config-keys
+ </value>
+ <description>A comma-separated or multi-line list of regular expressions to
+ match configuration keys that should be redacted where appropriate, for
example, when logging modified properties during a reconfiguration,
private credentials should not be logged.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344bea3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
index eedb9b2..ba08de4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
@@ -34,15 +34,30 @@ public class TestConfigRedactor {
private static final String ORIGINAL_VALUE = "Hello, World!";
@Test
- public void redact() throws Exception {
+ public void testRedactWithCoreDefault() throws Exception {
Configuration conf = new Configuration();
+ testRedact(conf);
+ }
+
+ @Test
+ public void testRedactNoCoreDefault() throws Exception {
+ Configuration conf = new Configuration(false);
+ testRedact(conf);
+ }
+
+ private void testRedact(Configuration conf) throws Exception {
ConfigRedactor redactor = new ConfigRedactor(conf);
String processedText;
List<String> sensitiveKeys = Arrays.asList(
"fs.s3a.secret.key",
+ "fs.s3a.bucket.BUCKET.secret.key",
"fs.s3n.awsSecretKey",
"fs.azure.account.key.abcdefg.blob.core.windows.net",
+ "fs.adl.oauth2.refresh.token",
+ "fs.adl.oauth2.credential",
+ "dfs.adls.oauth2.refresh.token",
+ "dfs.adls.oauth2.credential",
"dfs.webhdfs.oauth2.access.token",
"dfs.webhdfs.oauth2.refresh.token",
"ssl.server.keystore.keypassword",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org