You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2019/02/27 03:16:34 UTC
[hive] branch master updated: Revert "HIVE-21247: Webhcat beeline
in secure mode (Daniel Dai, reviewed by Thejas Nair)"
This is an automated email from the ASF dual-hosted git repository.
daijy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 77b9c3c Revert "HIVE-21247: Webhcat beeline in secure mode (Daniel Dai, reviewed by Thejas Nair)"
77b9c3c is described below
commit 77b9c3c79953121c19554a7de6953a6e21de9db6
Author: Daniel Dai <da...@gmail.com>
AuthorDate: Tue Feb 26 19:16:07 2019 -0800
Revert "HIVE-21247: Webhcat beeline in secure mode (Daniel Dai, reviewed by Thejas Nair)"
This reverts commit 3e9614eebd19f9843b28d97aaee1c3dfb815fb3d.
---
.../src/java/org/apache/hive/beeline/BeeLine.java | 4 --
hcatalog/webhcat/svr/pom.xml | 16 -----
.../apache/hive/hcatalog/templeton/AppConfig.java | 1 -
.../hive/hcatalog/templeton/HiveDelegator.java | 6 --
.../hcatalog/templeton/SecureProxySupport.java | 1 -
.../hive/hcatalog/templeton/tool/LaunchMapper.java | 20 +------
.../templeton/tool/TempletonControllerJob.java | 68 ++++------------------
.../java/org/apache/hive/jdbc/HiveConnection.java | 26 +--------
packaging/src/main/assembly/bin.xml | 3 -
pom.xml | 1 -
10 files changed, 14 insertions(+), 132 deletions(-)
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index cded55f..65eee2c 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -1203,10 +1203,6 @@ public class BeeLine implements Closeable {
if (password != null) {
jdbcConnectionParams.getSessionVars().put(JdbcConnectionParams.AUTH_PASSWD, password);
}
- String auth = cl.getOptionValue("a");
- if (auth != null) {
- jdbcConnectionParams.getSessionVars().put(JdbcConnectionParams.AUTH_TYPE, auth);
- }
mergedConnectionProperties =
HS2ConnectionFileUtils.mergeUserConnectionPropertiesAndBeelineSite(
userConnectionProperties, jdbcConnectionParams);
diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml
index 36d15cd..4dfade5 100644
--- a/hcatalog/webhcat/svr/pom.xml
+++ b/hcatalog/webhcat/svr/pom.xml
@@ -92,12 +92,6 @@
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<version>${jersey.version}</version>
- <exclusions>
- <exclusion>
- <groupId>javax.ws.rs</groupId>
- <artifactId>jsr311-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
@@ -150,11 +144,6 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.version}</version>
@@ -210,11 +199,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>javax.ws.rs</groupId>
- <artifactId>javax.ws.rs-api</artifactId>
- <version>${rs-api.version}</version>
- </dependency>
<!-- test inter-project -->
<dependency>
<groupId>org.apache.hive</groupId>
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
index b566cf8..1fd9e47 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
@@ -163,7 +163,6 @@ public class AppConfig extends Configuration {
* of escape/unescape methods in {@link org.apache.hadoop.util.StringUtils} in webhcat.
*/
public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
- public static final String HIVE_SERVER2_URL = "templeton.hive.hs2.url";
public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive";
public static final String SQOOP_PATH_NAME = "templeton.sqoop.path";
public static final String SQOOP_HOME_PATH = "templeton.sqoop.home";
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
index 3f679ac..3f1968d 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
@@ -28,7 +28,6 @@ import java.util.Map;
import org.apache.commons.exec.ExecuteException;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants;
import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
@@ -79,11 +78,6 @@ public class HiveDelegator extends LauncherDelegator {
args.add("-p");
args.add("default");
- if (UserGroupInformation.isSecurityEnabled()) {
- args.add("-a");
- args.add("delegationToken");
- }
-
//add mapreduce job tag placeholder
args.add("--hiveconf");
args.add(TempletonControllerJob.HIVE_QUERY_TAG_ARG_PLACEHOLDER);
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
index bbe5947..c503a7a 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
@@ -50,7 +50,6 @@ import org.apache.thrift.TException;
public class SecureProxySupport {
private Path tokenPath;
public static final String HCAT_SERVICE = "hcat";
- public static final String HIVE_SERVICE = "hive";
private final boolean isEnabled;
private String user;
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
index a776a0b..b1f4a6a 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
@@ -18,10 +18,6 @@
*/
package org.apache.hive.hcatalog.templeton.tool;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hive.hcatalog.templeton.SecureProxySupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -157,10 +153,10 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
env.put(pathVarName, paths);
}
}
- protected Process startJob(Context context, String jobId, String user, String overrideClasspath,
+ protected Process startJob(Configuration conf, String jobId, String user, String overrideClasspath,
LauncherDelegator.JobType jobType)
throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
+
copyLocal(COPY_NAME, conf);
String[] jarArgs = TempletonUtils.decodeArray(conf.get(JAR_ARGS_NAME));
@@ -178,16 +174,6 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary");
handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path");
if (jobType == LauncherDelegator.JobType.HIVE) {
- Credentials cred = new Credentials();
- Token<? extends TokenIdentifier> token = context.getCredentials().getToken(new
- Text(SecureProxySupport.HIVE_SERVICE));
- cred.addToken(new
- Text(SecureProxySupport.HIVE_SERVICE), token);
- File t = File.createTempFile("templeton", null);
- Path tokenPath = new Path(t.toURI());
- cred.writeTokenStorageFile(tokenPath, conf);
- env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION,
- tokenPath.toUri().getPath());
replaceJobTag(jarArgsList, JobSubmissionConstants.HIVE_QUERY_TAG_ARG_PLACEHOLDER,
JobSubmissionConstants.HIVE_QUERY_TAG, jobId);
} else {
@@ -419,7 +405,7 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
killLauncherChildJobs(conf, context.getJobID().toString());
// Start the job
- Process proc = startJob(context,
+ Process proc = startJob(conf,
context.getJobID().toString(),
conf.get("user.name"),
conf.get(OVERRIDE_CLASSPATH),
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
index 834b54b..bbb33cc 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
@@ -20,12 +20,8 @@ package org.apache.hive.hcatalog.templeton.tool;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
-import java.sql.DriverManager;
-import java.sql.SQLException;
import java.util.Arrays;
-import org.apache.hive.hcatalog.templeton.LauncherDelegator;
-import org.apache.hive.jdbc.HiveConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -146,43 +142,32 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi
Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token"));
job.getCredentials().addToken(new Text("mr token"), mrdt);
}
- LauncherDelegator.JobType jobType = LauncherDelegator.JobType.valueOf(conf.get(JOB_TYPE));
-
- String tokenStrForm = null;
- if (jobType == LauncherDelegator.JobType.HIVE) {
- tokenStrForm = addToken(job, user, SecureProxySupport.HIVE_SERVICE);
- } else {
- tokenStrForm = addToken(job, user, SecureProxySupport.HCAT_SERVICE);
- }
+ String metastoreTokenStrForm = addHMSToken(job, user);
job.submit();
JobID submittedJobId = job.getJobID();
- if(tokenStrForm != null) {
+ if(metastoreTokenStrForm != null) {
//so that it can be cancelled later from CompleteDelegator
DelegationTokenCache.getStringFormTokenCache().storeDelegationToken(
- submittedJobId.toString(), tokenStrForm);
- LOG.debug("Added delegation token for jobId=" + submittedJobId.toString() +
+ submittedJobId.toString(), metastoreTokenStrForm);
+ LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() +
" user=" + user);
}
return 0;
}
- private String addToken(Job job, String user, String type) throws IOException, InterruptedException,
+ private String addHMSToken(Job job, String user) throws IOException, InterruptedException,
TException {
if(!secureMetastoreAccess) {
return null;
}
Token<org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier> hiveToken =
new Token<org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier>();
- String tokenStrForm;
- if (type.equals(SecureProxySupport.HIVE_SERVICE)) {
- tokenStrForm = buildHS2DelegationToken(user);
- } else {
- tokenStrForm = buildHcatDelegationToken(user);
- }
- hiveToken.decodeFromUrlString(tokenStrForm);
- job.getCredentials().addToken(new Text(type), hiveToken);
- return tokenStrForm;
+ String metastoreTokenStrForm = buildHcatDelegationToken(user);
+ hiveToken.decodeFromUrlString(metastoreTokenStrForm);
+ job.getCredentials().addToken(new
+ Text(SecureProxySupport.HCAT_SERVICE), hiveToken);
+ return metastoreTokenStrForm;
}
private String buildHcatDelegationToken(String user) throws IOException, InterruptedException,
TException {
@@ -204,37 +189,4 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi
}
});
}
-
- private String buildHS2DelegationToken(String user) throws IOException, InterruptedException,
- TException {
- final HiveConf c = new HiveConf();
- LOG.debug("Creating hiveserver2 delegation token for user " + user);
- final UserGroupInformation ugi = UgiFactory.getUgi(user);
- UserGroupInformation real = ugi.getRealUser();
- return real.doAs(new PrivilegedExceptionAction<String>() {
- @Override
- public String run() throws IOException, TException, InterruptedException {
- try {
- Class.forName("org.apache.hive.jdbc.HiveDriver");
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- String hs2Url = appConf.get(AppConfig.HIVE_SERVER2_URL);
- final HiveConnection con;
- try {
- con = (HiveConnection) DriverManager.getConnection(hs2Url);
- } catch (SQLException e) {
- throw new IOException(e);
- }
- String token = ugi.doAs(new PrivilegedExceptionAction<String>() {
- @Override
- public String run() throws SQLException {
- String u = ugi.getUserName();
- return con.getDelegationToken(u,u);
- }
- });
- return token;
- }
- });
- }
}
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index 32a4761..8d5aa70 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -18,12 +18,6 @@
package org.apache.hive.jdbc;
-import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hive.service.rpc.thrift.TSetClientInfoResp;
import org.apache.hive.service.rpc.thrift.TSetClientInfoReq;
@@ -80,7 +74,6 @@ import javax.security.auth.Subject;
import javax.security.sasl.Sasl;
import javax.security.sasl.SaslException;
import java.io.BufferedReader;
-import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -766,23 +759,7 @@ public class HiveConnection implements java.sql.Connection {
if (JdbcConnectionParams.AUTH_TOKEN.equalsIgnoreCase(jdbcConnConf.get(JdbcConnectionParams.AUTH_TYPE))) {
// check delegation token in job conf if any
try {
- if (System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION) != null) {
- try {
- Credentials cred = new Credentials();
- DataInputStream dis = new DataInputStream(new FileInputStream(System.getenv(UserGroupInformation
- .HADOOP_TOKEN_FILE_LOCATION)));
- cred.readTokenStorageStream(dis);
- dis.close();
- Token<? extends TokenIdentifier> token = cred.getToken(new Text("hive"));
- tokenStr = token.encodeToUrlString();
- } catch (IOException e) {
- LOG.warn("Cannot get token from environment variable $HADOOP_TOKEN_FILE_LOCATION=" +
- System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION));
- }
- }
- if (tokenStr == null) {
- tokenStr = SessionUtils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN);
- }
+ tokenStr = SessionUtils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN);
} catch (IOException e) {
throw new SQLException("Error reading token ", e);
}
@@ -873,7 +850,6 @@ public class HiveConnection implements java.sql.Connection {
private boolean isKerberosAuthMode() {
return !JdbcConnectionParams.AUTH_SIMPLE.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE))
- && !JdbcConnectionParams.AUTH_TOKEN.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE))
&& sessConfMap.containsKey(JdbcConnectionParams.AUTH_PRINCIPAL);
}
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index 766161d..fceb1be 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -101,12 +101,9 @@
<useTransitiveFiltering>true</useTransitiveFiltering>
<excludes>
<exclude>org.apache.hadoop:*</exclude>
- <exclude>org.apache.hive:hive-jdbc:jar:standalone</exclude>
- <exclude>org.apache.httpcomponents:*</exclude>
</excludes>
<includes>
<include>org.apache.hive.hcatalog:hive-webhcat:*</include>
- <include>org.apache.hive:hive-jdbc:jar</include>
</includes>
</dependencySet>
<dependencySet>
diff --git a/pom.xml b/pom.xml
index 23c4413..c2d8641 100644
--- a/pom.xml
+++ b/pom.xml
@@ -217,7 +217,6 @@
<jsr305.version>3.0.0</jsr305.version>
<tephra.version>0.6.0</tephra.version>
<gson.version>2.2.4</gson.version>
- <rs-api.version>2.0.1</rs-api.version>
</properties>
<repositories>