You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/04/04 22:36:41 UTC
[06/50] [abbrv] hive git commit: HIVE-13008 - WebHcat DDL commands in
secure mode NPE when default FileSystem doesn't support delegation tokens
(Eugene Koifman, reviewed by Chris Nauroth, Thejas Nair)
HIVE-13008 - WebHcat DDL commands in secure mode NPE when default FileSystem doesn't support delegation tokens (Eugene Koifman, reviewed by Chris Nauroth, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ab095f0b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ab095f0b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ab095f0b
Branch: refs/heads/llap
Commit: ab095f0bc24447ab73843a1ae23a32f7b6c4bd1a
Parents: f9d1b6a
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Thu Mar 24 18:03:32 2016 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Thu Mar 24 18:03:32 2016 -0700
----------------------------------------------------------------------
.../hcatalog/templeton/SecureProxySupport.java | 46 ++++++++++++++------
1 file changed, 33 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ab095f0b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
index 2ac62c0..13f3c9b 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
@@ -20,10 +20,14 @@ package org.apache.hive.hcatalog.templeton;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -79,7 +83,7 @@ public class SecureProxySupport {
this.user = user;
File t = File.createTempFile("templeton", null);
tokenPath = new Path(t.toURI());
- Token fsToken = getFSDelegationToken(user, conf);
+ Token[] fsToken = getFSDelegationToken(user, conf);
String hcatTokenStr;
try {
hcatTokenStr = buildHcatDelegationToken(user);
@@ -130,11 +134,11 @@ public class SecureProxySupport {
}
}
- class TokenWrapper {
- Token<?> token;
+ private static class TokenWrapper {
+ Token<?>[] tokens = new Token<?>[0];
}
- private Token<?> getFSDelegationToken(String user,
+ private Token<?>[] getFSDelegationToken(String user,
final Configuration conf)
throws IOException, InterruptedException {
LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName());
@@ -142,18 +146,32 @@ public class SecureProxySupport {
final TokenWrapper twrapper = new TokenWrapper();
ugi.doAs(new PrivilegedExceptionAction<Object>() {
- public Object run() throws IOException {
- FileSystem fs = FileSystem.get(conf);
- //todo: according to JavaDoc this seems like private API: addDelegationToken should be used
- twrapper.token = fs.getDelegationToken(ugi.getShortUserName());
+ public Object run() throws IOException, URISyntaxException {
+ Credentials creds = new Credentials();
+ //get Tokens for default FS. Not all FSs support delegation tokens, e.g. WASB
+ collectTokens(FileSystem.get(conf), twrapper, creds, ugi.getShortUserName());
+ //get tokens for all other known FSs since Hive tables may result in different ones
+ //passing "creds" prevents duplicate tokens from being added
+ Collection<String> URIs = conf.getStringCollection("mapreduce.job.hdfs-servers");
+ for(String uri : URIs) {
+ LOG.debug("Getting tokens for " + uri);
+ collectTokens(FileSystem.get(new URI(uri), conf), twrapper, creds, ugi.getShortUserName());
+ }
return null;
}
});
- return twrapper.token;
-
+ return twrapper.tokens;
}
-
- private void writeProxyDelegationTokens(final Token<?> fsToken,
+ private static void collectTokens(FileSystem fs, TokenWrapper twrapper, Credentials creds, String userName) throws IOException {
+ Token[] tokens = fs.addDelegationTokens(userName, creds);
+ if(tokens != null && tokens.length > 0) {
+ twrapper.tokens = ArrayUtils.addAll(twrapper.tokens, tokens);
+ }
+ }
+ /**
+ * @param fsTokens not null
+ */
+ private void writeProxyDelegationTokens(final Token<?> fsTokens[],
final Token<?> msToken,
final Configuration conf,
String user,
@@ -168,7 +186,9 @@ public class SecureProxySupport {
ugi.doAs(new PrivilegedExceptionAction<Object>() {
public Object run() throws IOException {
Credentials cred = new Credentials();
- cred.addToken(fsToken.getService(), fsToken);
+ for(Token<?> fsToken : fsTokens) {
+ cred.addToken(fsToken.getService(), fsToken);
+ }
cred.addToken(msToken.getService(), msToken);
cred.writeTokenStorageFile(tokenPath, conf);
return null;