You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/08/14 14:00:57 UTC
[13/26] hadoop git commit: HADOOP-14260.
Configuration.dumpConfiguration should redact sensitive information.
Contributed by John Zhuge.
HADOOP-14260. Configuration.dumpConfiguration should redact sensitive information. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/582648be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/582648be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/582648be
Branch: refs/heads/HADOOP-13345
Commit: 582648befaf9908159f937d2cc8f549583a3483e
Parents: 4222c97
Author: John Zhuge <jz...@cloudera.com>
Authored: Thu Aug 10 16:28:22 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Aug 11 10:16:08 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 15 +++---
.../apache/hadoop/conf/TestConfiguration.java | 48 ++++++++++++++++++--
2 files changed, 53 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/582648be/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 65e8569..edaee68 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -3146,7 +3146,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("property");
- appendJSONProperty(dumpGenerator, config, propertyName);
+ appendJSONProperty(dumpGenerator, config, propertyName,
+ new ConfigRedactor(config));
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
@@ -3186,11 +3187,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
dumpGenerator.flush();
+ ConfigRedactor redactor = new ConfigRedactor(config);
synchronized (config) {
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
- appendJSONProperty(dumpGenerator,
- config,
- item.getKey().toString());
+ appendJSONProperty(dumpGenerator, config, item.getKey().toString(),
+ redactor);
}
}
dumpGenerator.writeEndArray();
@@ -3208,12 +3209,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @throws IOException
*/
private static void appendJSONProperty(JsonGenerator jsonGen,
- Configuration config, String name) throws IOException {
+ Configuration config, String name, ConfigRedactor redactor)
+ throws IOException {
// skip writing if given property name is empty or null
if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
jsonGen.writeStartObject();
jsonGen.writeStringField("key", name);
- jsonGen.writeStringField("value", config.get(name));
+ jsonGen.writeStringField("value",
+ redactor.redact(name, config.get(name)));
jsonGen.writeBooleanField("isFinal",
config.finalParameters.contains(name));
String[] resources = config.updatingResource.get(name);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/582648be/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 92d3290..91f25fa 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -49,6 +49,7 @@ import static org.junit.Assert.assertArrayEquals;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@@ -82,6 +83,11 @@ public class TestConfiguration extends TestCase {
/** Four apostrophes. */
public static final String ESCAPED = "''''";
+ private static final String SENSITIVE_CONFIG_KEYS =
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
+
+ private BufferedWriter out;
+
@Override
protected void setUp() throws Exception {
super.setUp();
@@ -90,6 +96,9 @@ public class TestConfiguration extends TestCase {
@Override
protected void tearDown() throws Exception {
+ if(out != null) {
+ out.close();
+ }
super.tearDown();
new File(CONFIG).delete();
new File(CONFIG2).delete();
@@ -878,8 +887,6 @@ public class TestConfiguration extends TestCase {
new File(new File(relConfig).getParent()).delete();
}
- BufferedWriter out;
-
public void testIntegerRanges() {
Configuration conf = new Configuration();
conf.set("first", "-100");
@@ -1787,8 +1794,41 @@ public class TestConfiguration extends TestCase {
assertEquals(fileResource.toString(),prop.getResource());
}
}
-
-
+
+ public void testDumpSensitiveProperty() throws IOException {
+ final String myPassword = "ThisIsMyPassword";
+ Configuration testConf = new Configuration(false);
+ out = new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.password", myPassword);
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ testConf.addResource(fileResource);
+
+ try (StringWriter outWriter = new StringWriter()) {
+ testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
+ Configuration.dumpConfiguration(testConf, "test.password", outWriter);
+ assertFalse(outWriter.toString().contains(myPassword));
+ }
+ }
+
+ public void testDumpSensitiveConfiguration() throws IOException {
+ final String myPassword = "ThisIsMyPassword";
+ Configuration testConf = new Configuration(false);
+ out = new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.password", myPassword);
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ testConf.addResource(fileResource);
+
+ try (StringWriter outWriter = new StringWriter()) {
+ testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
+ Configuration.dumpConfiguration(testConf, outWriter);
+ assertFalse(outWriter.toString().contains(myPassword));
+ }
+ }
+
public void testGetValByRegex() {
Configuration conf = new Configuration();
String key1 = "t.abc.key1";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org