You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jx...@apache.org on 2015/12/31 16:56:54 UTC
hive git commit: HIVE-11369: Mapjoins in HiveServer2 fail when
jmxremote is used (Jimmy, reviewed by Aihua)
Repository: hive
Updated Branches:
refs/heads/master 1a460b503 -> 5d7162e07
HIVE-11369: Mapjoins in HiveServer2 fail when jmxremote is used (Jimmy, reviewed by Aihua)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5d7162e0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5d7162e0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5d7162e0
Branch: refs/heads/master
Commit: 5d7162e072033948ad140b6dca1d1137588aa5cc
Parents: 1a460b5
Author: Jimmy Xiang <jx...@apache.org>
Authored: Tue Dec 29 15:51:54 2015 -0800
Committer: Jimmy Xiang <jx...@apache.org>
Committed: Wed Dec 30 15:00:09 2015 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/mr/MapRedTask.java | 2 +-
.../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 37 ++++++++++++++++----
2 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5d7162e0/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
index 7ec5ed1..5bc3d9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
@@ -228,7 +228,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
hadoopOpts = sb.toString();
// Inherit the environment variables
String[] env;
- Map<String, String> variables = new HashMap(System.getenv());
+ Map<String, String> variables = new HashMap<String, String>(System.getenv());
// The user can specify the hadoop memory
if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/5d7162e0/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
index f191572..ee99269 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
@@ -17,6 +17,11 @@
*/
package org.apache.hadoop.hive.ql.exec.mr;
+import static org.apache.hadoop.hive.ql.exec.mr.MapRedTask.HADOOP_CLIENT_OPTS;
+import static org.apache.hadoop.hive.ql.exec.mr.MapRedTask.HADOOP_MEM_KEY;
+import static org.apache.hadoop.hive.ql.exec.mr.MapRedTask.HADOOP_OPTS_KEY;
+import static org.apache.hadoop.hive.ql.exec.mr.MapRedTask.HIVE_SYS_PROP;
+
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
@@ -33,9 +38,6 @@ import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -49,6 +51,7 @@ import org.apache.hadoop.hive.ql.exec.BucketMatcher;
import org.apache.hadoop.hive.ql.exec.FetchOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.SecureCmdDoAs;
+import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -70,6 +73,8 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hive.common.util.StreamPrinter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* MapredLocalTask represents any local work (i.e.: client side work) that hive needs to
@@ -81,13 +86,13 @@ import org.apache.hive.common.util.StreamPrinter;
*/
public class MapredLocalTask extends Task<MapredLocalWork> implements Serializable {
+ private static final long serialVersionUID = 1L;
+
private final Map<String, FetchOperator> fetchOperators = new HashMap<String, FetchOperator>();
protected HadoopJobExecHelper jobExecHelper;
private JobConf job;
public static transient final Logger l4j = LoggerFactory.getLogger(MapredLocalTask.class);
- static final String HADOOP_MEM_KEY = "HADOOP_HEAPSIZE";
- static final String HADOOP_OPTS_KEY = "HADOOP_OPTS";
- static final String[] HIVE_SYS_PROP = {"build.dir", "build.dir.hive", "hive.query.id"};
+ static final String HIVE_LOCAL_TASK_CHILD_OPTS_KEY = "HIVE_LOCAL_TASK_CHILD_OPTS";
public static MemoryMXBean memoryMXBean;
private static final Logger LOG = LoggerFactory.getLogger(MapredLocalTask.class);
@@ -211,7 +216,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
hadoopOpts = sb.toString();
// Inherit the environment variables
String[] env;
- Map<String, String> variables = new HashMap(System.getenv());
+ Map<String, String> variables = new HashMap<String, String>(System.getenv());
// The user can specify the hadoop memory
// if ("local".equals(conf.getVar(HiveConf.ConfVars.HADOOPJT))) {
@@ -275,6 +280,24 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
secureDoAs.addEnv(variables);
}
+ // If HIVE_LOCAL_TASK_CHILD_OPTS is set, child VM environment setting
+ // HADOOP_CLIENT_OPTS will be replaced with HIVE_LOCAL_TASK_CHILD_OPTS.
+ // HADOOP_OPTS is updated too since HADOOP_CLIENT_OPTS is appended
+ // to HADOOP_OPTS in most cases. This way, the local task JVM can
+ // have different settings from those of HiveServer2.
+ if (variables.containsKey(HIVE_LOCAL_TASK_CHILD_OPTS_KEY)) {
+ String childOpts = variables.get(HIVE_LOCAL_TASK_CHILD_OPTS_KEY);
+ if (childOpts == null) {
+ childOpts = "";
+ }
+ String clientOpts = variables.put(HADOOP_CLIENT_OPTS, childOpts);
+ String tmp = variables.get(HADOOP_OPTS_KEY);
+ if (tmp != null && !StringUtils.isBlank(clientOpts)) {
+ tmp = tmp.replace(clientOpts, childOpts);
+ variables.put(HADOOP_OPTS_KEY, tmp);
+ }
+ }
+
env = new String[variables.size()];
int pos = 0;
for (Map.Entry<String, String> entry : variables.entrySet()) {