You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2017/03/28 21:09:50 UTC
[33/50] [abbrv] hive git commit: HIVE-15935: ACL is not set in ATS
data (Daniel Dai, reviewed by Siddharth Seth)
HIVE-15935: ACL is not set in ATS data (Daniel Dai, reviewed by Siddharth Seth)
Change-Id: I95734893c18cefe58731323ad33e6097118c3378
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fa53585a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fa53585a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fa53585a
Branch: refs/heads/branch-2.2
Commit: fa53585a891d88b30d88413c96d9881fdda73e7b
Parents: 38c9ac7
Author: Daniel Dai <da...@hortonworks.com>
Authored: Tue Feb 28 15:40:50 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 14:02:44 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/hooks/ATSHook.java | 88 +++++++++++++++++++-
.../hadoop/hive/ql/session/SessionState.java | 10 +++
2 files changed, 95 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fa53585a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 2dd03f5..218e0b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -33,6 +33,7 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
@@ -43,12 +44,16 @@ import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.parse.ExplainConfiguration;
import org.apache.hadoop.hive.ql.plan.ExplainWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hive.common.util.ShutdownHookManager;
+import org.apache.tez.dag.api.TezConfiguration;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +74,9 @@ public class ATSHook implements ExecuteWithHookContext {
private static TimelineClient timelineClient;
private enum EntityTypes { HIVE_QUERY_ID };
private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED };
+ private static final String ATS_DOMAIN_PREFIX = "hive_";
+ private static boolean defaultATSDomainCreated = false;
+ private static final String DEFAULT_ATS_DOMAIN = "hive_default_ats";
private enum OtherInfoTypes {
QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, SESSION_ID, THREAD_NAME, VERSION,
@@ -136,6 +144,78 @@ public class ATSHook implements ExecuteWithHookContext {
LOG.info("Created ATS Hook");
}
+ private void createTimelineDomain(String domainId, String readers, String writers) throws Exception {
+ TimelineDomain timelineDomain = new TimelineDomain();
+ timelineDomain.setId(domainId);
+ timelineDomain.setReaders(readers);
+ timelineDomain.setWriters(writers);
+ timelineClient.putDomain(timelineDomain);
+ LOG.info("ATS domain created:" + domainId + "(" + readers + "," + writers + ")");
+ }
+
+ private String createOrGetDomain(final HookContext hookContext) throws Exception {
+ final String domainId;
+ String domainReaders = null;
+ String domainWriters = null;
+ boolean create = false;
+ if (SessionState.get() != null) {
+ if (SessionState.get().getATSDomainId() == null) {
+ domainId = ATS_DOMAIN_PREFIX + hookContext.getSessionId();
+ // Create session domain if not present
+ if (SessionState.get().getATSDomainId() == null) {
+ String requestuser = hookContext.getUserName();
+ if (hookContext.getUserName() == null ){
+ requestuser = hookContext.getUgi().getShortUserName() ;
+ }
+ boolean addHs2User =
+ HiveConf.getBoolVar(hookContext.getConf(), ConfVars.HIVETEZHS2USERACCESS);
+
+ UserGroupInformation loginUserUgi = UserGroupInformation.getLoginUser();
+ String loginUser =
+ loginUserUgi == null ? null : loginUserUgi.getShortUserName();
+
+ // In Tez, TEZ_AM_VIEW_ACLS/TEZ_AM_MODIFY_ACLS is used as the base for Tez ATS ACLS,
+ // so if exists, honor it. So we get the same ACLS for Tez ATS entries and
+ // Hive entries
+ domainReaders = Utilities.getAclStringWithHiveModification(hookContext.getConf(),
+ TezConfiguration.TEZ_AM_VIEW_ACLS, addHs2User, requestuser, loginUser);
+
+ domainWriters = Utilities.getAclStringWithHiveModification(hookContext.getConf(),
+ TezConfiguration.TEZ_AM_MODIFY_ACLS, addHs2User, requestuser, loginUser);
+ SessionState.get().setATSDomainId(domainId);
+ create = true;
+ }
+ } else {
+ domainId = SessionState.get().getATSDomainId();
+ }
+ } else {
+ // SessionState is null, this is unlikely to happen, just in case
+ if (!defaultATSDomainCreated) {
+ domainReaders = domainWriters = UserGroupInformation.getCurrentUser().getShortUserName();
+ defaultATSDomainCreated = true;
+ create = true;
+ }
+ domainId = DEFAULT_ATS_DOMAIN;
+ }
+ if (create) {
+ final String readers = domainReaders;
+ final String writers = domainWriters;
+ // executor is single thread, so we can guarantee
+ // domain created before any ATS entries
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ createTimelineDomain(domainId, readers, writers);
+ } catch (Exception e) {
+ LOG.warn("Failed to create ATS domain " + domainId, e);
+ }
+ }
+ });
+ }
+ return domainId;
+ }
+
@Override
public void run(final HookContext hookContext) throws Exception {
final long currentTime = System.currentTimeMillis();
@@ -151,6 +231,7 @@ public class ATSHook implements ExecuteWithHookContext {
try {
setupAtsExecutor(conf);
+ final String domainId = createOrGetDomain(hookContext);
executor.submit(new Runnable() {
@Override
public void run() {
@@ -223,12 +304,12 @@ public class ATSHook implements ExecuteWithHookContext {
case POST_EXEC_HOOK:
fireAndForget(createPostHookEvent(queryId,
currentTime, user, requestuser, true, opId,
- durations));
+ durations, domainId));
break;
case ON_FAILURE_HOOK:
fireAndForget(createPostHookEvent(queryId,
currentTime, user, requestuser, false,
- opId, durations));
+ opId, durations, domainId));
break;
default:
// ignore
@@ -342,7 +423,7 @@ public class ATSHook implements ExecuteWithHookContext {
}
TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser, boolean success,
- String opId, Map<String, Long> durations) throws Exception {
+ String opId, Map<String, Long> durations, String domainId) throws Exception {
LOG.info("Received post-hook notification for :" + queryId);
TimelineEntity atsEntity = new TimelineEntity();
@@ -367,6 +448,7 @@ public class ATSHook implements ExecuteWithHookContext {
perfObj.put(entry.getKey(), entry.getValue());
}
atsEntity.addOtherInfo(OtherInfoTypes.PERF.name(), perfObj.toString());
+ atsEntity.setDomainId(domainId);
return atsEntity;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fa53585a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 847081b..6fc9055 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -283,6 +283,8 @@ public class SessionState {
private List<String> forwardedAddresses;
+ private String atsDomainId;
+
/**
* Get the lineage state stored in this session.
*
@@ -1251,6 +1253,14 @@ public class SessionState {
}
}
+ public String getATSDomainId() {
+ return atsDomainId;
+ }
+
+ public void setATSDomainId(String domainId) {
+ this.atsDomainId = domainId;
+ }
+
/**
* ResourceType.
*