You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/06/27 15:57:10 UTC

hive git commit: HIVE-14098: Logging task properties, and environment variables might contain passwords (Peter Vary, reviewed by Sergio Pena)

Repository: hive
Updated Branches:
  refs/heads/master bad467de7 -> 44bc8e71b


HIVE-14098: Logging task properties, and environment variables might contain passwords (Peter Vary, reviewed by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44bc8e71
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44bc8e71
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44bc8e71

Branch: refs/heads/master
Commit: 44bc8e71b6b7bd6a396873b72c2f1fc872782252
Parents: bad467d
Author: Peter Vary <pv...@cloudera.com>
Authored: Mon Jun 27 10:55:48 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Mon Jun 27 10:55:48 2016 -0500

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/exec/Utilities.java   | 23 +++++++++++++++++++-
 .../hadoop/hive/ql/exec/mr/MapredLocalTask.java |  2 +-
 .../ql/exec/spark/HiveSparkClientFactory.java   | 11 +++++-----
 .../hadoop/hive/ql/exec/TestUtilities.java      | 10 +++++++++
 4 files changed, 39 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/44bc8e71/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 528d663..3fab298 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -221,6 +221,13 @@ public final class Utilities {
   public static String REDUCENAME = "Reducer ";
 
   /**
+   * Constants for log masking
+   */
+  private static String KEY_TO_MASK_WITH = "password";
+  private static String MASKED_VALUE = "###_MASKED_###";
+
+
+  /**
    * ReduceField:
    * KEY: record key
    * VALUE: record value
@@ -3616,5 +3623,19 @@ public final class Utilities {
     return rowObjectInspector;
   }
 
-
+  /**
+   * Returns MASKED_VALUE if the key contains KEY_TO_MASK_WITH or the original property otherwise.
+   * Used to mask environment variables, and properties in logs which contain passwords
+   * @param key The property key to check
+   * @param value The original value of the property
+   * @return The masked property value
+   */
+  public static String maskIfPassword(String key, String value) {
+    if (key!=null && value!=null) {
+      if (key.toLowerCase().indexOf(KEY_TO_MASK_WITH) != -1) {
+        return MASKED_VALUE;
+      }
+    }
+    return value;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/44bc8e71/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
index 24bf506..a26d2b8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
@@ -309,7 +309,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
         String name = entry.getKey();
         String value = entry.getValue();
         env[pos++] = name + "=" + value;
-        LOG.debug("Setting env: " + env[pos-1]);
+        LOG.debug("Setting env: " + name + "=" + Utilities.maskIfPassword(name, value));
       }
 
       LOG.info("Executing: " + cmdLine);

http://git-wip-us.apache.org/repos/asf/hive/blob/44bc8e71/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
index b1df1d6..8e5c0da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
@@ -28,6 +28,7 @@ import java.util.Set;
 
 import org.apache.commons.compress.utils.CharsetNames;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -98,7 +99,7 @@ public class HiveSparkClientFactory {
             sparkConf.put(propertyName, properties.getProperty(propertyName));
             LOG.info(String.format(
               "load spark property from %s (%s -> %s).",
-              SPARK_DEFAULT_CONF_FILE, propertyName, value));
+              SPARK_DEFAULT_CONF_FILE, propertyName, Utilities.maskIfPassword(propertyName,value)));
           }
         }
       }
@@ -135,7 +136,7 @@ public class HiveSparkClientFactory {
         sparkConf.put(propertyName, value);
         LOG.info(String.format(
           "load spark property from hive configuration (%s -> %s).",
-          propertyName, value));
+          propertyName, Utilities.maskIfPassword(propertyName,value)));
       } else if (propertyName.startsWith("yarn") &&
         (sparkMaster.equals("yarn-client") || sparkMaster.equals("yarn-cluster"))) {
         String value = hiveConf.get(propertyName);
@@ -145,7 +146,7 @@ public class HiveSparkClientFactory {
         sparkConf.put("spark.hadoop." + propertyName, value);
         LOG.info(String.format(
           "load yarn property from hive configuration in %s mode (%s -> %s).",
-          sparkMaster, propertyName, value));
+          sparkMaster, propertyName, Utilities.maskIfPassword(propertyName,value)));
       } else if (propertyName.equals(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)) {
         String value = hiveConf.get(propertyName);
         if (value != null && !value.isEmpty()) {
@@ -158,7 +159,7 @@ public class HiveSparkClientFactory {
         String value = hiveConf.get(propertyName);
         sparkConf.put("spark.hadoop." + propertyName, value);
         LOG.info(String.format(
-          "load HBase configuration (%s -> %s).", propertyName, value));
+          "load HBase configuration (%s -> %s).", propertyName, Utilities.maskIfPassword(propertyName,value)));
       }
 
       if (RpcConfiguration.HIVE_SPARK_RSC_CONFIGS.contains(propertyName)) {
@@ -166,7 +167,7 @@ public class HiveSparkClientFactory {
         sparkConf.put(propertyName, value);
         LOG.info(String.format(
           "load RPC property from hive configuration (%s -> %s).",
-          propertyName, value));
+          propertyName, Utilities.maskIfPassword(propertyName,value)));
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/44bc8e71/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
index bb6a4e1..cc59f13 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
@@ -151,4 +151,14 @@ public class TestUtilities extends TestCase {
     Assert.assertEquals("(ds%3D1)000005", newTaskID);
   }
 
+  public void testMaskIfPassword() {
+    Assert.assertNull(Utilities.maskIfPassword("",null));
+    Assert.assertNull(Utilities.maskIfPassword(null,null));
+    Assert.assertEquals("test",Utilities.maskIfPassword(null,"test"));
+    Assert.assertEquals("test2",Utilities.maskIfPassword("any","test2"));
+    Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("password","test3"));
+    Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("a_passWord","test4"));
+    Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("password_a","test5"));
+    Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("a_PassWord_a","test6"));
+  }
 }