You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2019/01/07 14:33:06 UTC
hive git commit: HIVE-20914: MRScratchDir permission denied when
"hive.server2.enable.doAs",
"hive.exec.submitviachild" are set to "true" and impersonated/proxy user is
used (Denys Kuzmenko reviewed by Adam Szita and Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master ac73ad80b -> 5a70751ba
HIVE-20914: MRScratchDir permission denied when "hive.server2.enable.doAs", "hive.exec.submitviachild" are set to "true" and impersonated/proxy user is used (Denys Kuzmenko reviewed by Adam Szita and Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5a70751b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5a70751b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5a70751b
Branch: refs/heads/master
Commit: 5a70751ba2dd79025ce4c55c64ede2d70660fa8b
Parents: ac73ad8
Author: denys kuzmenko <dk...@cloudera.com>
Authored: Mon Jan 7 15:22:51 2019 +0100
Committer: Peter Vary <pv...@cloudera.com>
Committed: Mon Jan 7 15:22:51 2019 +0100
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/mr/MapRedTask.java | 18 ++++++-
.../hadoop/hive/ql/exec/mr/TestMapRedTask.java | 49 ++++++++++++++++++++
2 files changed, 66 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5a70751b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
index 8266906..0f594a1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
@@ -49,10 +49,16 @@ import org.apache.hadoop.hive.ql.plan.ReduceWork;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.shims.Utils;
import org.apache.hive.common.util.StreamPrinter;
import org.apache.hadoop.mapred.RunningJob;
+
+import com.google.common.annotations.VisibleForTesting;
+
import org.json.JSONException;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.PROXY;
+
/**
* Extension of ExecDriver:
* - can optionally spawn a map-reduce task from a separate jvm
@@ -71,6 +77,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
static final String HIVE_MAIN_CLIENT_DEBUG_OPTS = "HIVE_MAIN_CLIENT_DEBUG_OPTS";
static final String HIVE_CHILD_CLIENT_DEBUG_OPTS = "HIVE_CHILD_CLIENT_DEBUG_OPTS";
static final String[] HIVE_SYS_PROP = {"build.dir", "build.dir.hive", "hive.query.id"};
+ static final String HADOOP_PROXY_USER = "HADOOP_PROXY_USER";
private transient ContentSummary inputSummary = null;
private transient boolean runningViaChild = false;
@@ -267,6 +274,10 @@ public class MapRedTask extends ExecDriver implements Serializable {
configureDebugVariablesForChildJVM(variables);
}
+ if (PROXY == Utils.getUGI().getAuthenticationMethod()) {
+ variables.put(HADOOP_PROXY_USER, Utils.getUGI().getShortUserName());
+ }
+
env = new String[variables.size()];
int pos = 0;
for (Map.Entry<String, String> entry : variables.entrySet()) {
@@ -275,7 +286,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
env[pos++] = name + "=" + value;
}
// Run ExecDriver in another JVM
- executor = Runtime.getRuntime().exec(cmdLine, env, new File(workDir));
+ executor = spawn(cmdLine, workDir, env);
CachingPrintStream errPrintStream =
new CachingPrintStream(SessionState.getConsole().getChildErrStream());
@@ -323,6 +334,11 @@ public class MapRedTask extends ExecDriver implements Serializable {
}
}
+ @VisibleForTesting
+ Process spawn(String cmdLine, String workDir, String[] env) throws IOException {
+ return Runtime.getRuntime().exec(cmdLine, env, new File(workDir));
+ }
+
static void configureDebugVariablesForChildJVM(Map<String, String> environmentVariables) {
// this method contains various asserts to warn if environment variables are in a buggy state
assert environmentVariables.containsKey(HADOOP_CLIENT_OPTS)
http://git-wip-us.apache.org/repos/asf/hive/blob/5a70751b/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java
index 0693d24..c9b68fa 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java
@@ -17,16 +17,32 @@
*/
package org.apache.hadoop.hive.ql.exec.mr;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.PROXY;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.Arrays;
+import javax.security.auth.login.LoginException;
+
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
+import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.shims.Utils;
+
+import org.junit.Assert;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
public class TestMapRedTask {
@@ -44,4 +60,37 @@ public class TestMapRedTask {
verify(mockMetrics, never()).incrementCounter(MetricsConstant.HIVE_SPARK_TASKS);
}
+ @Test
+ public void mrTaskSumbitViaChildWithImpersonation() throws IOException, LoginException {
+ Utils.getUGI().setAuthenticationMethod(PROXY);
+
+ Context ctx = Mockito.mock(Context.class);
+ when(ctx.getLocalTmpPath()).thenReturn(new Path(System.getProperty("java.io.tmpdir")));
+
+ DriverContext dctx = new DriverContext(ctx);
+
+ QueryState queryState = new QueryState.Builder().build();
+ HiveConf conf= queryState.getConf();
+ conf.setBoolVar(HiveConf.ConfVars.SUBMITVIACHILD, true);
+
+ MapredWork mrWork = new MapredWork();
+ mrWork.setMapWork(Mockito.mock(MapWork.class));
+
+ MapRedTask mrTask = Mockito.spy(new MapRedTask());
+ mrTask.setWork(mrWork);
+
+ mrTask.initialize(queryState, null, dctx, null);
+
+ mrTask.jobExecHelper = Mockito.mock(HadoopJobExecHelper.class);
+ when(mrTask.jobExecHelper.progressLocal(Mockito.any(Process.class), Mockito.anyString())).thenReturn(0);
+
+ mrTask.execute(dctx);
+
+ ArgumentCaptor<String[]> captor = ArgumentCaptor.forClass(String[].class);
+ verify(mrTask).spawn(Mockito.anyString(), Mockito.anyString(), captor.capture());
+
+ String expected = "HADOOP_PROXY_USER=" + Utils.getUGI().getUserName();
+ Assert.assertTrue(Arrays.asList(captor.getValue()).contains(expected));
+ }
+
}