You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ct...@apache.org on 2016/09/28 15:17:03 UTC
hive git commit: Revert "HIVE-14426: Extensive logging on info level
in WebHCat (Peter Vary via Chaoyu Tang)"
Repository: hive
Updated Branches:
refs/heads/branch-2.1 80156443d -> 6bf2790d5
Revert "HIVE-14426: Extensive logging on info level in WebHCat (Peter Vary via Chaoyu Tang)"
This reverts commit 80156443db185ca85db0928e609f6cbe2f6ceb3f.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6bf2790d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6bf2790d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6bf2790d
Branch: refs/heads/branch-2.1
Commit: 6bf2790d5a64c448b554df85d9c184bfd8c32008
Parents: 8015644
Author: ctang <ct...@cloudera.com>
Authored: Wed Sep 28 11:15:50 2016 -0400
Committer: ctang <ct...@cloudera.com>
Committed: Wed Sep 28 11:15:50 2016 -0400
----------------------------------------------------------------------
.../org/apache/hadoop/hive/common/LogUtils.java | 22 -----------
.../org/apache/hadoop/hive/conf/HiveConf.java | 20 ++++++++--
.../apache/hadoop/hive/conf/HiveConfUtil.java | 39 +-------------------
.../apache/hadoop/hive/common/TestLogUtils.java | 34 -----------------
.../hive/hcatalog/templeton/AppConfig.java | 15 +-------
.../hcatalog/templeton/tool/TempletonUtils.java | 4 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 7 ++++
.../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 3 +-
.../ql/exec/spark/HiveSparkClientFactory.java | 11 +++---
9 files changed, 34 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
index c2a0d9a..599e798 100644
--- a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
@@ -42,12 +42,6 @@ public class LogUtils {
private static final String HIVE_EXEC_L4J = "hive-exec-log4j2.properties";
private static final Logger l4j = LoggerFactory.getLogger(LogUtils.class);
- /**
- * Constants for log masking
- */
- private static String KEY_TO_MASK_WITH = "password";
- private static String MASKED_VALUE = "###_MASKED_###";
-
@SuppressWarnings("serial")
public static class LogInitializationException extends Exception {
public LogInitializationException(String msg) {
@@ -177,20 +171,4 @@ public class LogUtils {
+ conf.getHiveSiteLocation().getPath());
}
}
-
- /**
- * Returns MASKED_VALUE if the key contains KEY_TO_MASK_WITH or the original property otherwise.
- * Used to mask environment variables, and properties in logs which contain passwords
- * @param key The property key to check
- * @param value The original value of the property
- * @return The masked property value
- */
- public static String maskIfPassword(String key, String value) {
- if (key!=null && value!=null) {
- if (key.toLowerCase().indexOf(KEY_TO_MASK_WITH) != -1) {
- return MASKED_VALUE;
- }
- }
- return value;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5a29684..178afbc 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3777,8 +3777,8 @@ public class HiveConf extends Configuration {
// setup list of conf vars that are not allowed to change runtime
setupRestrictList();
- hiddenSet.clear();
- hiddenSet.addAll(HiveConfUtil.getHiddenSet(this));
+ setupHiddenSet();
+
}
/**
@@ -4130,11 +4130,25 @@ public class HiveConf extends Configuration {
restrictList.add(ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST.varname);
}
+ private void setupHiddenSet() {
+ String hiddenListStr = this.getVar(ConfVars.HIVE_CONF_HIDDEN_LIST);
+ hiddenSet.clear();
+ if (hiddenListStr != null) {
+ for (String entry : hiddenListStr.split(",")) {
+ hiddenSet.add(entry.trim());
+ }
+ }
+ }
+
/**
* Strips hidden config entries from configuration
*/
public void stripHiddenConfigurations(Configuration conf) {
- HiveConfUtil.stripConfigurations(conf, hiddenSet);
+ for (String name : hiddenSet) {
+ if (conf.get(name) != null) {
+ conf.set(name, "");
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
index 16c2eaf..073a978 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
@@ -25,11 +25,9 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.StringTokenizer;
/**
@@ -61,42 +59,7 @@ public class HiveConfUtil {
dumpConfig(conf, sb);
return sb.append("END========\"new HiveConf()\"========\n");
}
-
- /**
- * Getting the set of the hidden configurations
- * @param configuration The original configuration
- * @return The list of the configuration values to hide
- */
- public static Set<String> getHiddenSet(Configuration configuration) {
- Set<String> hiddenSet = new HashSet<String>();
- String hiddenListStr = HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST);
- if (hiddenListStr != null) {
- for (String entry : hiddenListStr.split(",")) {
- hiddenSet.add(entry.trim());
- }
- }
- return hiddenSet;
- }
-
- /**
- * Strips hidden config entries from configuration
- * @param conf The configuration to strip from
- * @param hiddenSet The values to strip
- */
- public static void stripConfigurations(Configuration conf, Set<String> hiddenSet) {
- for (String name : hiddenSet) {
- if (conf.get(name) != null) {
- conf.set(name, "");
- }
- }
- }
-
- public static void dumpConfig(Configuration originalConf, StringBuilder sb) {
- Set<String> hiddenSet = getHiddenSet(originalConf);
- sb.append("Values omitted for security reason if present: ").append(hiddenSet).append("\n");
- Configuration conf = new Configuration(originalConf);
- stripConfigurations(conf, hiddenSet);
-
+ public static void dumpConfig(Configuration conf, StringBuilder sb) {
Iterator<Map.Entry<String, String>> configIter = conf.iterator();
List<Map.Entry<String, String>> configVals = new ArrayList<>();
while(configIter.hasNext()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/common/src/test/org/apache/hadoop/hive/common/TestLogUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/TestLogUtils.java b/common/src/test/org/apache/hadoop/hive/common/TestLogUtils.java
deleted file mode 100644
index 923ac2d..0000000
--- a/common/src/test/org/apache/hadoop/hive/common/TestLogUtils.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.apache.hadoop.hive.common;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestLogUtils {
- @Test
- public void testMaskIfPassword() {
- Assert.assertNull(LogUtils.maskIfPassword("",null));
- Assert.assertNull(LogUtils.maskIfPassword(null,null));
- Assert.assertEquals("test", LogUtils.maskIfPassword(null,"test"));
- Assert.assertEquals("test2", LogUtils.maskIfPassword("any","test2"));
- Assert.assertEquals("###_MASKED_###", LogUtils.maskIfPassword("password","test3"));
- Assert.assertEquals("###_MASKED_###", LogUtils.maskIfPassword("a_passWord","test4"));
- Assert.assertEquals("###_MASKED_###", LogUtils.maskIfPassword("password_a","test5"));
- Assert.assertEquals("###_MASKED_###", LogUtils.maskIfPassword("a_PassWord_a","test6"));
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
index 54d0907..dd1208b 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
@@ -200,19 +200,14 @@ public class AppConfig extends Configuration {
* been installed. We need pass some properties to that client to make sure it connects to the
* right Metastore, configures Tez, etc. Here we look for such properties in hive config,
* and set a comma-separated list of key values in {@link #HIVE_PROPS_NAME}.
- * The HIVE_CONF_HIDDEN_LIST should be handled separately too - this also should be copied from
- * the hive config to the webhcat config if not defined there.
* Note that the user may choose to set the same keys in HIVE_PROPS_NAME directly, in which case
* those values should take precedence.
*/
private void handleHiveProperties() {
HiveConf hiveConf = new HiveConf();//load hive-site.xml from classpath
List<String> interestingPropNames = Arrays.asList(
- HiveConf.ConfVars.METASTOREURIS.varname,
- HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname,
- HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI.varname,
- HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname,
- HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname);
+ "hive.metastore.uris","hive.metastore.sasl.enabled",
+ "hive.metastore.execute.setugi","hive.execution.engine");
//each items is a "key=value" format
List<String> webhcatHiveProps = new ArrayList<String>(hiveProps());
@@ -237,12 +232,6 @@ public class AppConfig extends Configuration {
hiveProps.append(hiveProps.length() > 0 ? "," : "").append(StringUtils.escapeString(whProp));
}
set(HIVE_PROPS_NAME, hiveProps.toString());
- // Setting the hidden list
- String hiddenProperties = hiveConf.get(HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname);
- if (this.get(HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname) == null
- && hiddenProperties!=null) {
- set(HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname, hiddenProperties);
- }
}
private static void logConfigLoadAttempt(String path) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
index f629a78..83584d3 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
@@ -42,7 +42,6 @@ import java.util.regex.Pattern;
import javax.ws.rs.core.UriBuilder;
-import org.apache.hadoop.hive.common.LogUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -484,8 +483,7 @@ public class TempletonUtils {
}
}
else {
- sb.append(propKey).append('=').append(LogUtils.maskIfPassword(propKey, map.get(propKey)));
- sb.append('\n');
+ sb.append(propKey).append('=').append(map.get(propKey)).append('\n');
}
}
return sb.append("END").append(header).append('\n');
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 3b6fdeb..202adf3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -225,6 +225,13 @@ public final class Utilities {
public static String REDUCENAME = "Reducer ";
/**
+ * Constants for log masking
+ */
+ private static String KEY_TO_MASK_WITH = "password";
+ private static String MASKED_VALUE = "###_MASKED_###";
+
+
+ /**
* ReduceField:
* KEY: record key
* VALUE: record value
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
index 127d3e6..d4b17d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
@@ -41,7 +41,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.common.io.CachingPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -310,7 +309,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
String name = entry.getKey();
String value = entry.getValue();
env[pos++] = name + "=" + value;
- LOG.debug("Setting env: " + name + "=" + LogUtils.maskIfPassword(name, value));
+ LOG.debug("Setting env: " + name + "=" + Utilities.maskIfPassword(name, value));
}
LOG.info("Executing: " + cmdLine);
http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
index 8867415..ed87adb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
@@ -28,7 +28,6 @@ import java.util.Set;
import org.apache.commons.compress.utils.CharsetNames;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.slf4j.Logger;
@@ -100,7 +99,7 @@ public class HiveSparkClientFactory {
sparkConf.put(propertyName, properties.getProperty(propertyName));
LOG.info(String.format(
"load spark property from %s (%s -> %s).",
- SPARK_DEFAULT_CONF_FILE, propertyName, LogUtils.maskIfPassword(propertyName,value)));
+ SPARK_DEFAULT_CONF_FILE, propertyName, Utilities.maskIfPassword(propertyName,value)));
}
}
}
@@ -137,7 +136,7 @@ public class HiveSparkClientFactory {
sparkConf.put(propertyName, value);
LOG.info(String.format(
"load spark property from hive configuration (%s -> %s).",
- propertyName, LogUtils.maskIfPassword(propertyName,value)));
+ propertyName, Utilities.maskIfPassword(propertyName,value)));
} else if (propertyName.startsWith("yarn") &&
(sparkMaster.equals("yarn-client") || sparkMaster.equals("yarn-cluster"))) {
String value = hiveConf.get(propertyName);
@@ -147,7 +146,7 @@ public class HiveSparkClientFactory {
sparkConf.put("spark.hadoop." + propertyName, value);
LOG.info(String.format(
"load yarn property from hive configuration in %s mode (%s -> %s).",
- sparkMaster, propertyName, LogUtils.maskIfPassword(propertyName,value)));
+ sparkMaster, propertyName, Utilities.maskIfPassword(propertyName,value)));
} else if (propertyName.equals(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)) {
String value = hiveConf.get(propertyName);
if (value != null && !value.isEmpty()) {
@@ -160,7 +159,7 @@ public class HiveSparkClientFactory {
String value = hiveConf.get(propertyName);
sparkConf.put("spark.hadoop." + propertyName, value);
LOG.info(String.format(
- "load HBase configuration (%s -> %s).", propertyName, LogUtils.maskIfPassword(propertyName,value)));
+ "load HBase configuration (%s -> %s).", propertyName, Utilities.maskIfPassword(propertyName,value)));
}
if (RpcConfiguration.HIVE_SPARK_RSC_CONFIGS.contains(propertyName)) {
@@ -168,7 +167,7 @@ public class HiveSparkClientFactory {
sparkConf.put(propertyName, value);
LOG.info(String.format(
"load RPC property from hive configuration (%s -> %s).",
- propertyName, LogUtils.maskIfPassword(propertyName,value)));
+ propertyName, Utilities.maskIfPassword(propertyName,value)));
}
}