You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/09/21 20:23:27 UTC
hive git commit: HIVE-14098: Logging task properties,
and environment variables might contain passwords (Peter Vary,
reviewed by Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/branch-2.1 b424fd097 -> 0646cc2d0
HIVE-14098: Logging task properties, and environment variables might contain passwords (Peter Vary, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0646cc2d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0646cc2d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0646cc2d
Branch: refs/heads/branch-2.1
Commit: 0646cc2d051b75bd3a042d1ea6d9f0292c505321
Parents: b424fd0
Author: Peter Vary <pv...@cloudera.com>
Authored: Wed Sep 21 15:22:41 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 21 15:22:41 2016 -0500
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/Utilities.java | 23 ++++++++++++++++++++
.../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 2 +-
.../ql/exec/spark/HiveSparkClientFactory.java | 11 +++++-----
.../hadoop/hive/ql/exec/TestUtilities.java | 12 ++++++++++
4 files changed, 42 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0646cc2d/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 8f7bbb2..202adf3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -225,6 +225,13 @@ public final class Utilities {
public static String REDUCENAME = "Reducer ";
/**
+ * Constants for log masking
+ */
+ private static String KEY_TO_MASK_WITH = "password";
+ private static String MASKED_VALUE = "###_MASKED_###";
+
+
+ /**
* ReduceField:
* KEY: record key
* VALUE: record value
@@ -3697,4 +3704,20 @@ public final class Utilities {
}
return result;
}
+
+ /**
+ * Returns MASKED_VALUE if the key contains KEY_TO_MASK_WITH or the original property otherwise.
+ * Used to mask environment variables, and properties in logs which contain passwords
+ * @param key The property key to check
+ * @param value The original value of the property
+ * @return The masked property value
+ */
+ public static String maskIfPassword(String key, String value) {
+ if (key!=null && value!=null) {
+ if (key.toLowerCase().indexOf(KEY_TO_MASK_WITH) != -1) {
+ return MASKED_VALUE;
+ }
+ }
+ return value;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0646cc2d/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
index f4d3d88..d4b17d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
@@ -309,7 +309,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
String name = entry.getKey();
String value = entry.getValue();
env[pos++] = name + "=" + value;
- LOG.debug("Setting env: " + env[pos-1]);
+ LOG.debug("Setting env: " + name + "=" + Utilities.maskIfPassword(name, value));
}
LOG.info("Executing: " + cmdLine);
http://git-wip-us.apache.org/repos/asf/hive/blob/0646cc2d/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
index b36c60e..ed87adb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.commons.compress.utils.CharsetNames;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,7 +99,7 @@ public class HiveSparkClientFactory {
sparkConf.put(propertyName, properties.getProperty(propertyName));
LOG.info(String.format(
"load spark property from %s (%s -> %s).",
- SPARK_DEFAULT_CONF_FILE, propertyName, value));
+ SPARK_DEFAULT_CONF_FILE, propertyName, Utilities.maskIfPassword(propertyName,value)));
}
}
}
@@ -135,7 +136,7 @@ public class HiveSparkClientFactory {
sparkConf.put(propertyName, value);
LOG.info(String.format(
"load spark property from hive configuration (%s -> %s).",
- propertyName, value));
+ propertyName, Utilities.maskIfPassword(propertyName,value)));
} else if (propertyName.startsWith("yarn") &&
(sparkMaster.equals("yarn-client") || sparkMaster.equals("yarn-cluster"))) {
String value = hiveConf.get(propertyName);
@@ -145,7 +146,7 @@ public class HiveSparkClientFactory {
sparkConf.put("spark.hadoop." + propertyName, value);
LOG.info(String.format(
"load yarn property from hive configuration in %s mode (%s -> %s).",
- sparkMaster, propertyName, value));
+ sparkMaster, propertyName, Utilities.maskIfPassword(propertyName,value)));
} else if (propertyName.equals(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)) {
String value = hiveConf.get(propertyName);
if (value != null && !value.isEmpty()) {
@@ -158,7 +159,7 @@ public class HiveSparkClientFactory {
String value = hiveConf.get(propertyName);
sparkConf.put("spark.hadoop." + propertyName, value);
LOG.info(String.format(
- "load HBase configuration (%s -> %s).", propertyName, value));
+ "load HBase configuration (%s -> %s).", propertyName, Utilities.maskIfPassword(propertyName,value)));
}
if (RpcConfiguration.HIVE_SPARK_RSC_CONFIGS.contains(propertyName)) {
@@ -166,7 +167,7 @@ public class HiveSparkClientFactory {
sparkConf.put(propertyName, value);
LOG.info(String.format(
"load RPC property from hive configuration (%s -> %s).",
- propertyName, value));
+ propertyName, Utilities.maskIfPassword(propertyName,value)));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0646cc2d/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
index d2060a1..b095608 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
@@ -246,4 +246,16 @@ public class TestUtilities {
FileSystem.getLocal(hconf).create(taskOutputPath).close();
return tempDirPath;
}
+
+ @Test
+ public void testMaskIfPassword() {
+ Assert.assertNull(Utilities.maskIfPassword("",null));
+ Assert.assertNull(Utilities.maskIfPassword(null,null));
+ Assert.assertEquals("test",Utilities.maskIfPassword(null,"test"));
+ Assert.assertEquals("test2",Utilities.maskIfPassword("any","test2"));
+ Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("password","test3"));
+ Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("a_passWord","test4"));
+ Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("password_a","test5"));
+ Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("a_PassWord_a","test6"));
+ }
}