You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/09/29 03:36:36 UTC
[45/48] hive git commit: HIVE-17621: Hive-site settings are ignored
during HCatInputFormat split-calculation (Chris Drome,
reviewed by Mithun Radhakrishnan)
HIVE-17621: Hive-site settings are ignored during HCatInputFormat split-calculation (Chris Drome, reviewed by Mithun Radhakrishnan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c32ff741
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c32ff741
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c32ff741
Branch: refs/heads/hive-14535
Commit: c32ff7413b3bcbf555ffc325d612cb853c7572d4
Parents: 6983d59
Author: Mithun RK <mi...@apache.org>
Authored: Wed Sep 27 11:10:36 2017 -0700
Committer: Mithun RK <mi...@apache.org>
Committed: Thu Sep 28 16:09:59 2017 -0700
----------------------------------------------------------------------
.../apache/hive/hcatalog/common/HCatUtil.java | 24 +++++++++++++++++---
.../hcatalog/mapreduce/HCatBaseInputFormat.java | 8 +++++--
2 files changed, 27 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c32ff741/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
index 107faf7..1e4ed75 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
@@ -33,6 +33,8 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
+import javax.security.auth.login.LoginException;
+
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -76,8 +78,6 @@ import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.security.auth.login.LoginException;
-
public class HCatUtil {
private static final Logger LOG = LoggerFactory.getLogger(HCatUtil.class);
@@ -642,6 +642,25 @@ public class HCatUtil {
return jobConf;
}
+ public static Map<String,String> getHCatKeyHiveConf(JobConf conf) {
+ Map<String,String> hiveProperties = new HashMap<String,String>();
+ if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) != null) {
+ try {
+ Properties properties = (Properties) HCatUtil.deserialize(
+ conf.get(HCatConstants.HCAT_KEY_HIVE_CONF));
+ for (Map.Entry<Object, Object> prop : properties.entrySet()) {
+ if (conf.get((String)prop.getKey()) == null) {
+ hiveProperties.put(prop.getKey().toString(), prop.getValue().toString());
+ }
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException(
+ "Failed to deserialize hive conf", e);
+ }
+ }
+ return hiveProperties;
+ }
+
public static void copyJobPropertiesToJobConf(
Map<String, String> jobProperties, JobConf jobConf) {
for (Map.Entry<String, String> entry : jobProperties.entrySet()) {
@@ -649,7 +668,6 @@ public class HCatUtil {
}
}
-
public static boolean isHadoop23() {
String version = org.apache.hadoop.util.VersionInfo.getVersion();
if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*"))
http://git-wip-us.apache.org/repos/asf/hive/blob/c32ff741/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
index 9caff7a..ec1e1ca 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java
@@ -126,16 +126,20 @@ public abstract class HCatBaseInputFormat
}
HiveStorageHandler storageHandler;
- JobConf jobConf;
+ Map<String,String> hiveProps = null;
//For each matching partition, call getSplits on the underlying InputFormat
for (PartInfo partitionInfo : partitionInfoList) {
- jobConf = HCatUtil.getJobConfFromContext(jobContext);
+ JobConf jobConf = HCatUtil.getJobConfFromContext(jobContext);
+ if (hiveProps == null) {
+ hiveProps = HCatUtil.getHCatKeyHiveConf(jobConf);
+ }
List<String> setInputPath = setInputPath(jobConf, partitionInfo.getLocation());
if (setInputPath.isEmpty()) {
continue;
}
Map<String, String> jobProperties = partitionInfo.getJobProperties();
+ HCatUtil.copyJobPropertiesToJobConf(hiveProps, jobConf);
HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf);
storageHandler = HCatUtil.getStorageHandler(