You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2018/10/12 12:28:25 UTC
hive git commit: HIVE-20644 : Avoid exposing sensitive infomation
through a Hive Runtime exception (Ashutosh Bapat reviewed by Thejas Nair,
Sankar Hariappan)
Repository: hive
Updated Branches:
refs/heads/master 1e45c4537 -> 24f7d2473
HIVE-20644 : Avoid exposing sensitive infomation through a Hive Runtime exception (Ashutosh Bapat reviewed by Thejas Nair, Sankar Hariappan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24f7d247
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24f7d247
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24f7d247
Branch: refs/heads/master
Commit: 24f7d2473c64053247c539af41288cf0e9110917
Parents: 1e45c45
Author: Ashutosh Bapat <ab...@hortonworks.com>
Authored: Fri Oct 12 07:28:21 2018 -0500
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Fri Oct 12 07:28:21 2018 -0500
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FunctionRegistry.java | 7 ++--
.../apache/hadoop/hive/ql/exec/MapOperator.java | 10 ++++--
.../hadoop/hive/ql/exec/mr/ExecReducer.java | 9 +++--
.../ql/exec/spark/SparkReduceRecordHandler.java | 38 +++++++++++++-------
.../hive/ql/exec/tez/ReduceRecordSource.java | 9 +++--
5 files changed, 53 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 0bc8d84..b7ca7c7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -1115,8 +1115,11 @@ public final class FunctionRegistry {
String detailedMsg = e instanceof java.lang.reflect.InvocationTargetException ?
e.getCause().getMessage() : e.getMessage();
- throw new HiveException("Unable to execute method " + m + " with arguments "
- + argumentString + ":" + detailedMsg, e);
+ // Log the arguments into a debug message for the ease of debugging. But when exposed through
+ // an error message they can leak sensitive information, even to the client application.
+ LOG.trace("Unable to execute method " + m + " with arguments "
+ + argumentString);
+ throw new HiveException("Unable to execute method " + m + ":" + detailedMsg, e);
}
return o;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
index b9986d3..1cbc272 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
@@ -562,9 +562,15 @@ public class MapOperator extends AbstractMapOperator {
}
if (row == null) {
deserialize_error_count.set(deserialize_error_count.get() + 1);
- throw new HiveException("Hive Runtime Error while processing writable " + message, e);
+ LOG.trace("Hive Runtime Error while processing writable " + message);
+ throw new HiveException("Hive Runtime Error while processing writable", e);
}
- throw new HiveException("Hive Runtime Error while processing row " + message, e);
+
+ // Log the contents of the row that caused exception so that it's available for debugging. But
+ // when exposed through an error message it can leak sensitive information, even to the
+ // client application.
+ LOG.trace("Hive Runtime Error while processing row " + message);
+ throw new HiveException("Hive Runtime Error while processing row", e);
}
}
rowsForwarded(childrenDone, 1);
http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
index 829006d..e106bc9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
@@ -240,8 +240,13 @@ public class ExecReducer extends MapReduceBase implements Reducer {
rowString = "[Error getting row data with exception " +
StringUtils.stringifyException(e2) + " ]";
}
- throw new HiveException("Hive Runtime Error while processing row (tag="
- + tag + ") " + rowString, e);
+
+ // Log the contents of the row that caused exception so that it's available for debugging. But
+ // when exposed through an error message it can leak sensitive information, even to the
+ // client application.
+ LOG.trace("Hive Runtime Error while processing row (tag="
+ + tag + ") " + rowString);
+ throw new HiveException("Hive Runtime Error while processing row", e);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
index 6a7e1df..20e7ea0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
@@ -346,11 +346,14 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
try {
keyObject = inputKeyDeserializer.deserialize(keyWritable);
} catch (Exception e) {
- throw new HiveException(
- "Hive Runtime Error: Unable to deserialize reduce input key from "
+ // Log the input key which caused exception so that it's available for debugging. But when
+ // exposed through an error message it can leak sensitive information, even to the client
+ // application.
+ LOG.trace("Hive Runtime Error: Unable to deserialize reduce input key from "
+ Utilities.formatBinaryString(keyWritable.get(), 0,
keyWritable.getSize()) + " with properties "
- + keyTableDesc.getProperties(), e);
+ + keyTableDesc.getProperties());
+ throw new HiveException("Hive Runtime Error: Unable to deserialize reduce input key ", e);
}
groupKey.set(keyWritable.get(), 0, keyWritable.getSize());
@@ -384,13 +387,16 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
try {
valueObject[tag] = inputValueDeserializer[tag].deserialize(valueWritable);
} catch (SerDeException e) {
- throw new HiveException(
- "Hive Runtime Error: Unable to deserialize reduce input value (tag="
+ // Log the input value which caused exception so that it's available for debugging. But when
+ // exposed through an error message it can leak sensitive information, even to the client
+ // application.
+ LOG.trace("Hive Runtime Error: Unable to deserialize reduce input value (tag="
+ tag
+ ") from "
+ Utilities.formatBinaryString(valueWritable.get(), 0,
valueWritable.getSize()) + " with properties "
- + valueTableDesc[tag].getProperties(), e);
+ + valueTableDesc[tag].getProperties());
+ throw new HiveException("Hive Runtime Error: Unable to deserialize reduce input value ", e);
}
row.clear();
row.add(keyObject);
@@ -408,8 +414,12 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
rowString = "[Error getting row data with exception "
+ StringUtils.stringifyException(e2) + " ]";
}
- throw new HiveException("Error while processing row (tag="
- + tag + ") " + rowString, e);
+
+ // Log contents of the row which caused exception so that it's available for debugging. But
+ // when exposed through an error message it can leak sensitive information, even to the
+ // client application.
+ LOG.trace("Hive exception while processing row (tag=" + tag + ") " + rowString);
+ throw new HiveException("Error while processing row ", e);
}
}
return true; // give me more
@@ -570,10 +580,14 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
try {
return inputValueDeserializer[tag].deserialize(valueWritable);
} catch (SerDeException e) {
- throw new HiveException("Error: Unable to deserialize reduce input value (tag="
- + tag + ") from "
- + Utilities.formatBinaryString(valueWritable.getBytes(), 0, valueWritable.getLength())
- + " with properties " + valueTableDesc[tag].getProperties(), e);
+ // Log the input value which caused exception so that it's available for debugging. But when
+ // exposed through an error message it can leak sensitive information, even to the client
+ // application.
+ LOG.trace("Error: Unable to deserialize reduce input value (tag=" + tag + ") from " +
+ Utilities.formatBinaryString(valueWritable.getBytes(), 0,
+ valueWritable.getLength()) +
+ " with properties " + valueTableDesc[tag].getProperties());
+ throw new HiveException("Error: Unable to deserialize reduce input value ", e);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
index 5698639..72446af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
@@ -369,8 +369,13 @@ public class ReduceRecordSource implements RecordSource {
rowString = "[Error getting row data with exception "
+ StringUtils.stringifyException(e2) + " ]";
}
- throw new HiveException("Hive Runtime Error while processing row (tag="
- + tag + ") " + rowString, e);
+
+ // Log the contents of the row that caused exception so that it's available for debugging. But
+ // when exposed through an error message it can leak sensitive information, even to the
+ // client application.
+ l4j.trace("Hive Runtime Error while processing row (tag="
+ + tag + ") " + rowString);
+ throw new HiveException("Hive Runtime Error while processing row", e);
}
}
}