You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2019/01/23 13:54:18 UTC
hive git commit: HIVE-21116: HADOOP_CREDSTORE_PASSWORD is not
populated under yarn.app.mapreduce.am.admin.user.env (Denys Kuzmenko via
Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master 2ffca04a8 -> 31bdf16e5
HIVE-21116: HADOOP_CREDSTORE_PASSWORD is not populated under yarn.app.mapreduce.am.admin.user.env (Denys Kuzmenko via Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/31bdf16e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/31bdf16e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/31bdf16e
Branch: refs/heads/master
Commit: 31bdf16e5b2948019927e61a5d79bbac9e6743ec
Parents: 2ffca04
Author: denys kuzmenko <dk...@cloudera.com>
Authored: Wed Jan 23 14:53:15 2019 +0100
Committer: Peter Vary <pv...@cloudera.com>
Committed: Wed Jan 23 14:53:15 2019 +0100
----------------------------------------------------------------------
.../apache/hadoop/hive/conf/HiveConfUtil.java | 16 ++++++-----
.../hive/common/util/HiveStringUtils.java | 28 +++++++++++++-------
.../ql/exec/TestHiveCredentialProviders.java | 23 +++++++++++++++-
3 files changed, 49 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/31bdf16e/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
index ae6fa43..ebe6423 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
@@ -42,6 +42,9 @@ import java.util.Set;
import java.util.StringTokenizer;
import java.util.stream.Stream;
+import static org.apache.hive.common.util.HiveStringUtils.COMMA;
+import static org.apache.hive.common.util.HiveStringUtils.EQUALS;
+
/**
* Hive Configuration utils
*/
@@ -205,7 +208,7 @@ public class HiveConfUtil {
Stream.of(
JobConf.MAPRED_MAP_TASK_ENV,
JobConf.MAPRED_REDUCE_TASK_ENV,
- "yarn.app.mapreduce.am.admin.user.env")
+ MRJobConfig.MR_AM_ADMIN_USER_ENV)
.forEach(property -> {
addKeyValuePair(jobConf, property,
@@ -215,7 +218,7 @@ public class HiveConfUtil {
// Hide sensitive configuration values from MR HistoryUI by telling MR to redact the following list.
jobConf.set(MRJobConfig.MR_JOB_REDACTED_PROPERTIES,
- StringUtils.join(redactedProperties, ","));
+ StringUtils.join(redactedProperties, COMMA));
}
}
}
@@ -241,14 +244,13 @@ public class HiveConfUtil {
return null;
}
- private static void addKeyValuePair(Configuration jobConf, String property, String keyName,
- String newKeyValue) {
+ private static void addKeyValuePair(Configuration jobConf, String property, String keyName, String newKeyValue) {
String existingValue = jobConf.get(property);
- if (existingValue == null) {
- jobConf.set(property, (keyName + "=" + newKeyValue));
+
+ if (StringUtils.isBlank(existingValue)) {
+ jobConf.set(property, (keyName + EQUALS + newKeyValue));
return;
}
-
String propertyValue = HiveStringUtils.insertValue(keyName, newKeyValue, existingValue);
jobConf.set(property, propertyValue);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/31bdf16e/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
index a4923f9..196b9c4 100644
--- a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
+++ b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
@@ -510,33 +510,41 @@ public class HiveStringUtils {
}
/**
- * In a given string of comma-separated key=value pairs insert a new value of a given key
+ * In a given string of comma-separated key=value pairs associates the specified value with
+ * the specified key.
+ * If the `string` previously contained a mapping for the key, the old value is replaced.
*
- * @param key The key whose value needs to be replaced
- * @param newValue The new value of the key
+ * @param key key with which the specified value is to be associated
+ * @param value value to be associated with the specified key
* @param strKvPairs Comma separated key=value pairs Eg: "k1=v1, k2=v2, k3=v3"
- * @return Comma separated string of key=value pairs with the new value for key keyName
+ * @return Updated comma separated string of key=value pairs
*/
- public static String insertValue(String key, String newValue,
- String strKvPairs) {
+ public static String insertValue(String key, String value, String strKvPairs) {
+ boolean keyNotFound = true;
+
String[] keyValuePairs = HiveStringUtils.split(strKvPairs);
StringBuilder sb = new StringBuilder();
+
for (int i = 0; i < keyValuePairs.length; i++) {
String[] pair = HiveStringUtils.split(keyValuePairs[i], ESCAPE_CHAR, EQUALS);
if (pair.length != 2) {
throw new RuntimeException("Error parsing the keyvalue pair " + keyValuePairs[i]);
}
- sb.append(pair[0]);
- sb.append(EQUALS);
+ sb.append(pair[0]).append(EQUALS);
if (pair[0].equals(key)) {
- sb.append(newValue);
+ sb.append(value);
+ keyNotFound = false;
} else {
sb.append(pair[1]);
}
- if (i < (keyValuePairs.length - 1)) {
+ if (i < (keyValuePairs.length - 1) || keyNotFound) {
sb.append(COMMA);
}
}
+
+ if (keyNotFound) {
+ sb.append(key).append(EQUALS).append(value);
+ }
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/31bdf16e/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java
index 4f49190..6030055 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java
@@ -50,7 +50,8 @@ public class TestHiveCredentialProviders {
private static final Collection<String> REDACTED_PROPERTIES = Arrays.asList(
JobConf.MAPRED_MAP_TASK_ENV,
- JobConf.MAPRED_REDUCE_TASK_ENV);
+ JobConf.MAPRED_REDUCE_TASK_ENV,
+ MRJobConfig.MR_AM_ADMIN_USER_ENV);
private Configuration jobConf;
@@ -102,6 +103,9 @@ public class TestHiveCredentialProviders {
Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf(
jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+ Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf(
+ jobConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+
Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES)
.containsAll(REDACTED_PROPERTIES));
}
@@ -126,6 +130,9 @@ public class TestHiveCredentialProviders {
Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf(
jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+ Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf(
+ jobConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+
Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES)
.containsAll(REDACTED_PROPERTIES));
}
@@ -146,6 +153,9 @@ public class TestHiveCredentialProviders {
Assert.assertNull(getValueFromJobConf(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV),
HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+ Assert.assertNull(getValueFromJobConf(jobConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV),
+ HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+
REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse(
jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES)
.contains(property)));
@@ -169,6 +179,9 @@ public class TestHiveCredentialProviders {
Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf(
jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+ Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf(
+ jobConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+
Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES)
.containsAll(REDACTED_PROPERTIES));
}
@@ -186,6 +199,7 @@ public class TestHiveCredentialProviders {
jobConf.get(HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG));
Assert.assertNull(jobConf.get(JobConf.MAPRED_MAP_TASK_ENV));
Assert.assertNull(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV));
+ Assert.assertNull(jobConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse(
jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES)
@@ -199,6 +213,7 @@ public class TestHiveCredentialProviders {
jobConf.get(HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG));
Assert.assertNull(jobConf.get(JobConf.MAPRED_MAP_TASK_ENV));
Assert.assertNull(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV));
+ Assert.assertNull(jobConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse(
jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES)
@@ -223,6 +238,9 @@ public class TestHiveCredentialProviders {
assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf(
jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+ Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf(
+ jobConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+
Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES)
.containsAll(REDACTED_PROPERTIES));
}
@@ -243,6 +261,9 @@ public class TestHiveCredentialProviders {
assertNull(getValueFromJobConf(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV),
HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+ assertNull(getValueFromJobConf(jobConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV),
+ HADOOP_CREDENTIAL_PASSWORD_ENVVAR));
+
REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse(
jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES)
.contains(property)));