You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jx...@apache.org on 2015/12/31 16:56:54 UTC

hive git commit: HIVE-11369: Mapjoins in HiveServer2 fail when jmxremote is used (Jimmy, reviewed by Aihua)

Repository: hive
Updated Branches:
  refs/heads/master 1a460b503 -> 5d7162e07


HIVE-11369: Mapjoins in HiveServer2 fail when jmxremote is used (Jimmy, reviewed by Aihua)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5d7162e0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5d7162e0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5d7162e0

Branch: refs/heads/master
Commit: 5d7162e072033948ad140b6dca1d1137588aa5cc
Parents: 1a460b5
Author: Jimmy Xiang <jx...@apache.org>
Authored: Tue Dec 29 15:51:54 2015 -0800
Committer: Jimmy Xiang <jx...@apache.org>
Committed: Wed Dec 30 15:00:09 2015 -0800

----------------------------------------------------------------------
 .../hadoop/hive/ql/exec/mr/MapRedTask.java      |  2 +-
 .../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 37 ++++++++++++++++----
 2 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5d7162e0/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
index 7ec5ed1..5bc3d9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
@@ -228,7 +228,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
       hadoopOpts = sb.toString();
       // Inherit the environment variables
       String[] env;
-      Map<String, String> variables = new HashMap(System.getenv());
+      Map<String, String> variables = new HashMap<String, String>(System.getenv());
       // The user can specify the hadoop memory
 
       if (ShimLoader.getHadoopShims().isLocalMode(conf)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/5d7162e0/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
index f191572..ee99269 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
@@ -17,6 +17,11 @@
  */
 package org.apache.hadoop.hive.ql.exec.mr;
 
+import static org.apache.hadoop.hive.ql.exec.mr.MapRedTask.HADOOP_CLIENT_OPTS;
+import static org.apache.hadoop.hive.ql.exec.mr.MapRedTask.HADOOP_MEM_KEY;
+import static org.apache.hadoop.hive.ql.exec.mr.MapRedTask.HADOOP_OPTS_KEY;
+import static org.apache.hadoop.hive.ql.exec.mr.MapRedTask.HIVE_SYS_PROP;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -33,9 +38,6 @@ import java.util.Properties;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
@@ -49,6 +51,7 @@ import org.apache.hadoop.hive.ql.exec.BucketMatcher;
 import org.apache.hadoop.hive.ql.exec.FetchOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.SecureCmdDoAs;
+import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -70,6 +73,8 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hive.common.util.StreamPrinter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * MapredLocalTask represents any local work (i.e.: client side work) that hive needs to
@@ -81,13 +86,13 @@ import org.apache.hive.common.util.StreamPrinter;
  */
 public class MapredLocalTask extends Task<MapredLocalWork> implements Serializable {
 
+  private static final long serialVersionUID = 1L;
+
   private final Map<String, FetchOperator> fetchOperators = new HashMap<String, FetchOperator>();
   protected HadoopJobExecHelper jobExecHelper;
   private JobConf job;
   public static transient final Logger l4j = LoggerFactory.getLogger(MapredLocalTask.class);
-  static final String HADOOP_MEM_KEY = "HADOOP_HEAPSIZE";
-  static final String HADOOP_OPTS_KEY = "HADOOP_OPTS";
-  static final String[] HIVE_SYS_PROP = {"build.dir", "build.dir.hive", "hive.query.id"};
+  static final String HIVE_LOCAL_TASK_CHILD_OPTS_KEY = "HIVE_LOCAL_TASK_CHILD_OPTS";
   public static MemoryMXBean memoryMXBean;
   private static final Logger LOG = LoggerFactory.getLogger(MapredLocalTask.class);
 
@@ -211,7 +216,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
       hadoopOpts = sb.toString();
       // Inherit the environment variables
       String[] env;
-      Map<String, String> variables = new HashMap(System.getenv());
+      Map<String, String> variables = new HashMap<String, String>(System.getenv());
       // The user can specify the hadoop memory
 
       // if ("local".equals(conf.getVar(HiveConf.ConfVars.HADOOPJT))) {
@@ -275,6 +280,24 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
         secureDoAs.addEnv(variables);
       }
 
+      // If HIVE_LOCAL_TASK_CHILD_OPTS is set, child VM environment setting
+      // HADOOP_CLIENT_OPTS will be replaced with HIVE_LOCAL_TASK_CHILD_OPTS.
+      // HADOOP_OPTS is updated too since HADOOP_CLIENT_OPTS is appended
+      // to HADOOP_OPTS in most cases. This way, the local task JVM can
+      // have different settings from those of HiveServer2.
+      if (variables.containsKey(HIVE_LOCAL_TASK_CHILD_OPTS_KEY)) {
+        String childOpts = variables.get(HIVE_LOCAL_TASK_CHILD_OPTS_KEY);
+        if (childOpts == null) {
+          childOpts = "";
+        }
+        String clientOpts = variables.put(HADOOP_CLIENT_OPTS, childOpts);
+        String tmp = variables.get(HADOOP_OPTS_KEY);
+        if (tmp != null && !StringUtils.isBlank(clientOpts)) {
+          tmp = tmp.replace(clientOpts, childOpts);
+          variables.put(HADOOP_OPTS_KEY, tmp);
+        }
+      }
+
       env = new String[variables.size()];
       int pos = 0;
       for (Map.Entry<String, String> entry : variables.entrySet()) {