You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2019/03/01 17:58:33 UTC
[hive] branch master updated: HIVE-21247: Webhcat beeline in secure
mode (Daniel Dai, reviewed by Thejas Nair)
This is an automated email from the ASF dual-hosted git repository.
daijy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8b2c6f0 HIVE-21247: Webhcat beeline in secure mode (Daniel Dai, reviewed by Thejas Nair)
8b2c6f0 is described below
commit 8b2c6f0cf28bdd7ce3fea5c4b191910064cae1c3
Author: Daniel Dai <da...@gmail.com>
AuthorDate: Fri Mar 1 09:57:55 2019 -0800
HIVE-21247: Webhcat beeline in secure mode (Daniel Dai, reviewed by Thejas Nair)
Signed-off-by: Thejas M Nair <th...@hortonworks.com>
---
.../src/java/org/apache/hive/beeline/BeeLine.java | 4 ++
hcatalog/webhcat/svr/pom.xml | 34 +++++++++++
.../apache/hive/hcatalog/templeton/AppConfig.java | 1 +
.../hive/hcatalog/templeton/HiveDelegator.java | 6 ++
.../hcatalog/templeton/SecureProxySupport.java | 1 +
.../hive/hcatalog/templeton/tool/LaunchMapper.java | 20 ++++++-
.../templeton/tool/TempletonControllerJob.java | 68 ++++++++++++++++++----
.../java/org/apache/hive/jdbc/HiveConnection.java | 26 ++++++++-
packaging/src/main/assembly/bin.xml | 3 +
pom.xml | 1 +
10 files changed, 150 insertions(+), 14 deletions(-)
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index 54edfeb..82077cc 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -1203,6 +1203,10 @@ public class BeeLine implements Closeable {
if (password != null) {
jdbcConnectionParams.getSessionVars().put(JdbcConnectionParams.AUTH_PASSWD, password);
}
+ String auth = cl.getOptionValue("a");
+ if (auth != null) {
+ jdbcConnectionParams.getSessionVars().put(JdbcConnectionParams.AUTH_TYPE, auth);
+ }
mergedConnectionProperties =
HS2ConnectionFileUtils.mergeUserConnectionPropertiesAndBeelineSite(
userConnectionProperties, jdbcConnectionParams);
diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml
index 4dfade5..75f1c70 100644
--- a/hcatalog/webhcat/svr/pom.xml
+++ b/hcatalog/webhcat/svr/pom.xml
@@ -92,21 +92,45 @@
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<version>${jersey.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>${jersey.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-servlet</artifactId>
<version>${jersey.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.sun.jersey.contribs</groupId>
<artifactId>wadl-resourcedoc-doclet</artifactId>
<version>${wadl-resourcedoc-doclet.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -144,6 +168,11 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.version}</version>
@@ -199,6 +228,11 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>javax.ws.rs-api</artifactId>
+ <version>${rs-api.version}</version>
+ </dependency>
<!-- test inter-project -->
<dependency>
<groupId>org.apache.hive</groupId>
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
index 1fd9e47..b566cf8 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
@@ -163,6 +163,7 @@ public class AppConfig extends Configuration {
* of escape/unescape methods in {@link org.apache.hadoop.util.StringUtils} in webhcat.
*/
public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
+ public static final String HIVE_SERVER2_URL = "templeton.hive.hs2.url";
public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive";
public static final String SQOOP_PATH_NAME = "templeton.sqoop.path";
public static final String SQOOP_HOME_PATH = "templeton.sqoop.home";
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
index 3f1968d..3f679ac 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
@@ -28,6 +28,7 @@ import java.util.Map;
import org.apache.commons.exec.ExecuteException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants;
import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
@@ -78,6 +79,11 @@ public class HiveDelegator extends LauncherDelegator {
args.add("-p");
args.add("default");
+ if (UserGroupInformation.isSecurityEnabled()) {
+ args.add("-a");
+ args.add("delegationToken");
+ }
+
//add mapreduce job tag placeholder
args.add("--hiveconf");
args.add(TempletonControllerJob.HIVE_QUERY_TAG_ARG_PLACEHOLDER);
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
index c503a7a..bbe5947 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
@@ -50,6 +50,7 @@ import org.apache.thrift.TException;
public class SecureProxySupport {
private Path tokenPath;
public static final String HCAT_SERVICE = "hcat";
+ public static final String HIVE_SERVICE = "hive";
private final boolean isEnabled;
private String user;
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
index b1f4a6a..a776a0b 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
@@ -18,6 +18,10 @@
*/
package org.apache.hive.hcatalog.templeton.tool;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hive.hcatalog.templeton.SecureProxySupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -153,10 +157,10 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
env.put(pathVarName, paths);
}
}
- protected Process startJob(Configuration conf, String jobId, String user, String overrideClasspath,
+ protected Process startJob(Context context, String jobId, String user, String overrideClasspath,
LauncherDelegator.JobType jobType)
throws IOException, InterruptedException {
-
+ Configuration conf = context.getConfiguration();
copyLocal(COPY_NAME, conf);
String[] jarArgs = TempletonUtils.decodeArray(conf.get(JAR_ARGS_NAME));
@@ -174,6 +178,16 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary");
handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path");
if (jobType == LauncherDelegator.JobType.HIVE) {
+ Credentials cred = new Credentials();
+ Token<? extends TokenIdentifier> token = context.getCredentials().getToken(new
+ Text(SecureProxySupport.HIVE_SERVICE));
+ cred.addToken(new
+ Text(SecureProxySupport.HIVE_SERVICE), token);
+ File t = File.createTempFile("templeton", null);
+ Path tokenPath = new Path(t.toURI());
+ cred.writeTokenStorageFile(tokenPath, conf);
+ env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION,
+ tokenPath.toUri().getPath());
replaceJobTag(jarArgsList, JobSubmissionConstants.HIVE_QUERY_TAG_ARG_PLACEHOLDER,
JobSubmissionConstants.HIVE_QUERY_TAG, jobId);
} else {
@@ -405,7 +419,7 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
killLauncherChildJobs(conf, context.getJobID().toString());
// Start the job
- Process proc = startJob(conf,
+ Process proc = startJob(context,
context.getJobID().toString(),
conf.get("user.name"),
conf.get(OVERRIDE_CLASSPATH),
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
index bbb33cc..834b54b 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
@@ -20,8 +20,12 @@ package org.apache.hive.hcatalog.templeton.tool;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.sql.DriverManager;
+import java.sql.SQLException;
import java.util.Arrays;
+import org.apache.hive.hcatalog.templeton.LauncherDelegator;
+import org.apache.hive.jdbc.HiveConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -142,32 +146,43 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi
Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token"));
job.getCredentials().addToken(new Text("mr token"), mrdt);
}
- String metastoreTokenStrForm = addHMSToken(job, user);
+ LauncherDelegator.JobType jobType = LauncherDelegator.JobType.valueOf(conf.get(JOB_TYPE));
+
+ String tokenStrForm = null;
+ if (jobType == LauncherDelegator.JobType.HIVE) {
+ tokenStrForm = addToken(job, user, SecureProxySupport.HIVE_SERVICE);
+ } else {
+ tokenStrForm = addToken(job, user, SecureProxySupport.HCAT_SERVICE);
+ }
job.submit();
JobID submittedJobId = job.getJobID();
- if(metastoreTokenStrForm != null) {
+ if(tokenStrForm != null) {
//so that it can be cancelled later from CompleteDelegator
DelegationTokenCache.getStringFormTokenCache().storeDelegationToken(
- submittedJobId.toString(), metastoreTokenStrForm);
- LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() +
+ submittedJobId.toString(), tokenStrForm);
+ LOG.debug("Added delegation token for jobId=" + submittedJobId.toString() +
" user=" + user);
}
return 0;
}
- private String addHMSToken(Job job, String user) throws IOException, InterruptedException,
+ private String addToken(Job job, String user, String type) throws IOException, InterruptedException,
TException {
if(!secureMetastoreAccess) {
return null;
}
Token<org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier> hiveToken =
new Token<org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier>();
- String metastoreTokenStrForm = buildHcatDelegationToken(user);
- hiveToken.decodeFromUrlString(metastoreTokenStrForm);
- job.getCredentials().addToken(new
- Text(SecureProxySupport.HCAT_SERVICE), hiveToken);
- return metastoreTokenStrForm;
+ String tokenStrForm;
+ if (type.equals(SecureProxySupport.HIVE_SERVICE)) {
+ tokenStrForm = buildHS2DelegationToken(user);
+ } else {
+ tokenStrForm = buildHcatDelegationToken(user);
+ }
+ hiveToken.decodeFromUrlString(tokenStrForm);
+ job.getCredentials().addToken(new Text(type), hiveToken);
+ return tokenStrForm;
}
private String buildHcatDelegationToken(String user) throws IOException, InterruptedException,
TException {
@@ -189,4 +204,37 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi
}
});
}
+
+ private String buildHS2DelegationToken(String user) throws IOException, InterruptedException,
+ TException {
+ final HiveConf c = new HiveConf();
+ LOG.debug("Creating hiveserver2 delegation token for user " + user);
+ final UserGroupInformation ugi = UgiFactory.getUgi(user);
+ UserGroupInformation real = ugi.getRealUser();
+ return real.doAs(new PrivilegedExceptionAction<String>() {
+ @Override
+ public String run() throws IOException, TException, InterruptedException {
+ try {
+ Class.forName("org.apache.hive.jdbc.HiveDriver");
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ String hs2Url = appConf.get(AppConfig.HIVE_SERVER2_URL);
+ final HiveConnection con;
+ try {
+ con = (HiveConnection) DriverManager.getConnection(hs2Url);
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ String token = ugi.doAs(new PrivilegedExceptionAction<String>() {
+ @Override
+ public String run() throws SQLException {
+ String u = ugi.getUserName();
+ return con.getDelegationToken(u,u);
+ }
+ });
+ return token;
+ }
+ });
+ }
}
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index 8d5aa70..32a4761 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -18,6 +18,12 @@
package org.apache.hive.jdbc;
+import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hive.service.rpc.thrift.TSetClientInfoResp;
import org.apache.hive.service.rpc.thrift.TSetClientInfoReq;
@@ -74,6 +80,7 @@ import javax.security.auth.Subject;
import javax.security.sasl.Sasl;
import javax.security.sasl.SaslException;
import java.io.BufferedReader;
+import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -759,7 +766,23 @@ public class HiveConnection implements java.sql.Connection {
if (JdbcConnectionParams.AUTH_TOKEN.equalsIgnoreCase(jdbcConnConf.get(JdbcConnectionParams.AUTH_TYPE))) {
// check delegation token in job conf if any
try {
- tokenStr = SessionUtils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN);
+ if (System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION) != null) {
+ try {
+ Credentials cred = new Credentials();
+ DataInputStream dis = new DataInputStream(new FileInputStream(System.getenv(UserGroupInformation
+ .HADOOP_TOKEN_FILE_LOCATION)));
+ cred.readTokenStorageStream(dis);
+ dis.close();
+ Token<? extends TokenIdentifier> token = cred.getToken(new Text("hive"));
+ tokenStr = token.encodeToUrlString();
+ } catch (IOException e) {
+ LOG.warn("Cannot get token from environment variable $HADOOP_TOKEN_FILE_LOCATION=" +
+ System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION));
+ }
+ }
+ if (tokenStr == null) {
+ tokenStr = SessionUtils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN);
+ }
} catch (IOException e) {
throw new SQLException("Error reading token ", e);
}
@@ -850,6 +873,7 @@ public class HiveConnection implements java.sql.Connection {
private boolean isKerberosAuthMode() {
return !JdbcConnectionParams.AUTH_SIMPLE.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE))
+ && !JdbcConnectionParams.AUTH_TOKEN.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE))
&& sessConfMap.containsKey(JdbcConnectionParams.AUTH_PRINCIPAL);
}
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index fceb1be..766161d 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -101,9 +101,12 @@
<useTransitiveFiltering>true</useTransitiveFiltering>
<excludes>
<exclude>org.apache.hadoop:*</exclude>
+ <exclude>org.apache.hive:hive-jdbc:jar:standalone</exclude>
+ <exclude>org.apache.httpcomponents:*</exclude>
</excludes>
<includes>
<include>org.apache.hive.hcatalog:hive-webhcat:*</include>
+ <include>org.apache.hive:hive-jdbc:jar</include>
</includes>
</dependencySet>
<dependencySet>
diff --git a/pom.xml b/pom.xml
index c2d8641..23c4413 100644
--- a/pom.xml
+++ b/pom.xml
@@ -217,6 +217,7 @@
<jsr305.version>3.0.0</jsr305.version>
<tephra.version>0.6.0</tephra.version>
<gson.version>2.2.4</gson.version>
+ <rs-api.version>2.0.1</rs-api.version>
</properties>
<repositories>