You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jz...@apache.org on 2017/08/11 18:00:51 UTC
[3/3] hadoop git commit: HADOOP-14260.
Configuration.dumpConfiguration should redact sensitive information.
Contributed by John Zhuge.
HADOOP-14260. Configuration.dumpConfiguration should redact sensitive information. Contributed by John Zhuge.
(cherry picked from commit 582648befaf9908159f937d2cc8f549583a3483e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e208a4e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e208a4e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e208a4e7
Branch: refs/heads/branch-2
Commit: e208a4e7ee5725b910a7e799066000cacf855218
Parents: 76c4aee
Author: John Zhuge <jz...@apache.org>
Authored: Fri Aug 11 10:54:41 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Aug 11 10:54:41 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 15 +++---
.../apache/hadoop/conf/TestConfiguration.java | 48 ++++++++++++++++++--
2 files changed, 53 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e208a4e7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 89714bf..42aa8f6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -3062,7 +3062,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("property");
- appendJSONProperty(dumpGenerator, config, propertyName);
+ appendJSONProperty(dumpGenerator, config, propertyName,
+ new ConfigRedactor(config));
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
@@ -3102,11 +3103,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
dumpGenerator.flush();
+ ConfigRedactor redactor = new ConfigRedactor(config);
synchronized (config) {
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
- appendJSONProperty(dumpGenerator,
- config,
- item.getKey().toString());
+ appendJSONProperty(dumpGenerator, config, item.getKey().toString(),
+ redactor);
}
}
dumpGenerator.writeEndArray();
@@ -3124,12 +3125,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @throws IOException
*/
private static void appendJSONProperty(JsonGenerator jsonGen,
- Configuration config, String name) throws IOException {
+ Configuration config, String name, ConfigRedactor redactor)
+ throws IOException {
// skip writing if given property name is empty or null
if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
jsonGen.writeStartObject();
jsonGen.writeStringField("key", name);
- jsonGen.writeStringField("value", config.get(name));
+ jsonGen.writeStringField("value",
+ redactor.redact(name, config.get(name)));
jsonGen.writeBooleanField("isFinal",
config.finalParameters.contains(name));
String[] resources = config.updatingResource.get(name);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e208a4e7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 24a49ad..66f717d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -48,6 +48,7 @@ import static org.junit.Assert.assertArrayEquals;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@@ -83,6 +84,11 @@ public class TestConfiguration extends TestCase {
/** Four apostrophes. */
public static final String ESCAPED = "''''";
+ private static final String SENSITIVE_CONFIG_KEYS =
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
+
+ private BufferedWriter out;
+
@Override
protected void setUp() throws Exception {
super.setUp();
@@ -91,6 +97,9 @@ public class TestConfiguration extends TestCase {
@Override
protected void tearDown() throws Exception {
+ if(out != null) {
+ out.close();
+ }
super.tearDown();
new File(CONFIG).delete();
new File(CONFIG2).delete();
@@ -791,8 +800,6 @@ public class TestConfiguration extends TestCase {
new File(new File(relConfig).getParent()).delete();
}
- BufferedWriter out;
-
public void testIntegerRanges() {
Configuration conf = new Configuration();
conf.set("first", "-100");
@@ -1654,8 +1661,41 @@ public class TestConfiguration extends TestCase {
assertEquals(fileResource.toString(),prop.getResource());
}
}
-
-
+
+ public void testDumpSensitiveProperty() throws IOException {
+ final String myPassword = "ThisIsMyPassword";
+ Configuration testConf = new Configuration(false);
+ out = new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.password", myPassword);
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ testConf.addResource(fileResource);
+
+ try (StringWriter outWriter = new StringWriter()) {
+ testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
+ Configuration.dumpConfiguration(testConf, "test.password", outWriter);
+ assertFalse(outWriter.toString().contains(myPassword));
+ }
+ }
+
+ public void testDumpSensitiveConfiguration() throws IOException {
+ final String myPassword = "ThisIsMyPassword";
+ Configuration testConf = new Configuration(false);
+ out = new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.password", myPassword);
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ testConf.addResource(fileResource);
+
+ try (StringWriter outWriter = new StringWriter()) {
+ testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
+ Configuration.dumpConfiguration(testConf, outWriter);
+ assertFalse(outWriter.toString().contains(myPassword));
+ }
+ }
+
public void testGetValByRegex() {
Configuration conf = new Configuration();
String key1 = "t.abc.key1";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org