You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ai...@apache.org on 2017/04/04 23:29:44 UTC
[1/2] hive git commit: HIVE-16335: Beeline user HS2 connection file
should use /etc/hive/conf instead of /etc/conf/hive (Vihang Karajgaonkar,
reviewed by Aihua Xu)
Repository: hive
Updated Branches:
refs/heads/master 23ac04d3b -> 4e60ea3f7
HIVE-16335: Beeline user HS2 connection file should use /etc/hive/conf instead of /etc/conf/hive (Vihang Karajgaonkar, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c7a44eb7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c7a44eb7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c7a44eb7
Branch: refs/heads/master
Commit: c7a44eb707084c65722a9779d3fc32746b36ed09
Parents: 23ac04d
Author: Aihua Xu <ai...@apache.org>
Authored: Tue Apr 4 19:12:50 2017 -0400
Committer: Aihua Xu <ai...@apache.org>
Committed: Tue Apr 4 19:28:16 2017 -0400
----------------------------------------------------------------------
.../hive/beeline/hs2connection/UserHS2ConnectionFileParser.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c7a44eb7/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java
index 93a6231..7d7d9ae 100644
--- a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java
+++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java
@@ -44,7 +44,7 @@ public class UserHS2ConnectionFileParser implements HS2ConnectionFileParser {
+ (System.getProperty("os.name").toLowerCase().indexOf("windows") != -1 ? "" : ".")
+ "beeline" + File.separator;
public static final String ETC_HIVE_CONF_LOCATION =
- File.separator + "etc" + File.separator + "conf" + File.separator + "hive";
+ File.separator + "etc" + File.separator + "hive" + File.separator + "conf";
private final List<String> locations = new ArrayList<>();
private static final Logger log = LoggerFactory.getLogger(UserHS2ConnectionFileParser.class);
[2/2] hive git commit: HIVE-16297: Improving hive logging
configuration variables (Vihang Karajgaonkar,
reviewed by Peter Vary & Aihua Xu)
Posted by ai...@apache.org.
HIVE-16297: Improving hive logging configuration variables (Vihang Karajgaonkar, reviewed by Peter Vary & Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4e60ea3f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4e60ea3f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4e60ea3f
Branch: refs/heads/master
Commit: 4e60ea3f786ed597b0594c2452dd7d64c44645b9
Parents: c7a44eb
Author: Aihua Xu <ai...@apache.org>
Authored: Tue Apr 4 19:14:28 2017 -0400
Committer: Aihua Xu <ai...@apache.org>
Committed: Tue Apr 4 19:28:46 2017 -0400
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/conf/HiveConfUtil.java | 13 ++++++++++++-
common/src/java/org/apache/hive/http/ConfServlet.java | 10 +++++++---
.../apache/hadoop/hive/ql/exec/FileSinkOperator.java | 10 ++++------
.../hive/ql/exec/spark/RemoteHiveSparkClient.java | 9 +++------
4 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
index 9ba08e5..dc02803 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
@@ -94,11 +94,22 @@ public class HiveConfUtil {
public static void stripConfigurations(Configuration conf, Set<String> hiddenSet) {
for (String name : hiddenSet) {
if (conf.get(name) != null) {
- conf.set(name, "");
+ conf.set(name, StringUtils.EMPTY);
}
}
}
+ /**
+ * Searches the given configuration object and replaces all the configuration values for keys
+ * defined hive.conf.hidden.list by empty String
+ *
+ * @param conf - Configuration object which needs to be modified to remove sensitive keys
+ */
+ public static void stripConfigurations(Configuration conf) {
+ Set<String> hiddenSet = getHiddenSet(conf);
+ stripConfigurations(conf, hiddenSet);
+ }
+
public static void dumpConfig(Configuration originalConf, StringBuilder sb) {
Set<String> hiddenSet = getHiddenSet(originalConf);
sb.append("Values omitted for security reason if present: ").append(hiddenSet).append("\n");
http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/common/src/java/org/apache/hive/http/ConfServlet.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/http/ConfServlet.java b/common/src/java/org/apache/hive/http/ConfServlet.java
index 253df4f..856a5d2 100644
--- a/common/src/java/org/apache/hive/http/ConfServlet.java
+++ b/common/src/java/org/apache/hive/http/ConfServlet.java
@@ -26,6 +26,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
/**
* A servlet to print out the running configuration data.
@@ -81,11 +82,14 @@ public class ConfServlet extends HttpServlet {
* Guts of the servlet - extracted for easy testing.
*/
static void writeResponse(Configuration conf, Writer out, String format)
- throws IOException, BadFormatException {
+ throws IOException, BadFormatException {
+ //redact the sensitive information from the configuration values
+ Configuration hconf = new Configuration(conf);
+ HiveConfUtil.stripConfigurations(hconf);
if (FORMAT_JSON.equals(format)) {
- Configuration.dumpConfiguration(conf, out);
+ Configuration.dumpConfiguration(hconf, out);
} else if (FORMAT_XML.equals(format)) {
- conf.writeXml(out);
+ hconf.writeXml(out);
} else {
throw new BadFormatException("Bad format: " + format);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index a9d03d0..4d727ba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -463,13 +464,10 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
private void logOutputFormatError(Configuration hconf, HiveException ex) {
- StringWriter errorWriter = new StringWriter();
+ StringBuilder errorWriter = new StringBuilder();
errorWriter.append("Failed to create output format; configuration: ");
- try {
- Configuration.dumpConfiguration(hconf, errorWriter);
- } catch (IOException ex2) {
- errorWriter.append("{ failed to dump configuration: " + ex2.getMessage() + " }");
- }
+ // redact sensitive information before logging
+ HiveConfUtil.dumpConfig(hconf, errorWriter);
Properties tdp = null;
if (this.conf.getTableInfo() != null
&& (tdp = this.conf.getTableInfo().getProperties()) != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index 4c69899..4ca8f93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -356,12 +356,9 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
private void logConfigurations(JobConf localJobConf) {
if (LOG.isInfoEnabled()) {
LOG.info("Logging job configuration: ");
- StringWriter outWriter = new StringWriter();
- try {
- Configuration.dumpConfiguration(localJobConf, outWriter);
- } catch (IOException e) {
- LOG.warn("Error logging job configuration", e);
- }
+ StringBuilder outWriter = new StringBuilder();
+ // redact sensitive information before logging
+ HiveConfUtil.dumpConfig(localJobConf, outWriter);
LOG.info(outWriter.toString());
}
}