You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2019/10/07 09:26:07 UTC
[hive] branch master updated: HIVE-22282: Obtain LLAP delegation
token only when LLAP is configured for Kerberos authentication (Denys
Kuzmenko, reviewed by Adam Szita)
This is an automated email from the ASF dual-hosted git repository.
szita pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2d99f78 HIVE-22282: Obtain LLAP delegation token only when LLAP is configured for Kerberos authentication (Denys Kuzmenko, reviewed by Adam Szita)
2d99f78 is described below
commit 2d99f7863f9da49ef56a85ae156f4bd4b9d5802b
Author: Denys Kuzmenko <dk...@cloudera.com>
AuthorDate: Mon Oct 7 11:24:59 2019 +0200
HIVE-22282: Obtain LLAP delegation token only when LLAP is configured for Kerberos authentication (Denys Kuzmenko, reviewed by Adam Szita)
---
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 9 +++++----
.../apache/hadoop/hive/registry/impl/ZookeeperUtils.java | 7 +++----
.../hadoop/hive/registry/impl/TestZookeeperUtils.java | 14 ++++++--------
.../apache/hadoop/hive/ql/exec/tez/TezSessionState.java | 6 +++++-
.../apache/hadoop/hive/metastore/conf/MetastoreConf.java | 4 ++++
.../hive/metastore/security/ZooKeeperTokenStore.java | 15 ++++++---------
6 files changed, 29 insertions(+), 26 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 7f7e77d..afee315 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2583,10 +2583,8 @@ public class HiveConf extends Configuration {
"read MM tables with original files will fail. The default in Hive 3.0 is false."),
// Zookeeper related configs
- HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION("hive.security.zookeeper.authentication",
- "DEFAULT", new StringSet("DEFAULT", "SIMPLE"),
- "Whether the authentication type for Zookeeper is different from the cluster wide\n" +
- "`hadoop.security.authentication` configuration. This could be useful when cluster\n" +
+ HIVE_ZOOKEEPER_USE_KERBEROS("hive.zookeeper.kerberos.enabled", true,
+ "If ZooKeeper is configured for Kerberos authentication. This could be useful when cluster\n" +
"is kerberized, but Zookeeper is not."),
HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "",
@@ -4212,6 +4210,9 @@ public class HiveConf extends Configuration {
"hive.llap.queue.metrics.percentiles.intervals"),
LLAP_IO_THREADPOOL_SIZE("hive.llap.io.threadpool.size", 10,
"Specify the number of threads to use for low-level IO thread pool."),
+ LLAP_USE_KERBEROS("hive.llap.kerberos.enabled", true,
+ "If LLAP is configured for Kerberos authentication. This could be useful when cluster\n" +
+ "is kerberized, but LLAP is not."),
LLAP_KERBEROS_PRINCIPAL(HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME, "",
"The name of the LLAP daemon's service principal."),
LLAP_KERBEROS_KEYTAB_FILE("hive.llap.daemon.keytab.file", "",
diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java
index 7a4274e..2ac0fbe 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java
@@ -14,7 +14,6 @@
package org.apache.hadoop.hive.registry.impl;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,12 +54,12 @@ public class ZookeeperUtils {
}
/**
- * Check if Kerberos is enabled.
+ * Check if Kerberos authentication is enabled.
*/
public static boolean isKerberosEnabled(Configuration conf) {
try {
- return UserGroupInformation.getLoginUser().isFromKeytab() && !AuthenticationMethod.SIMPLE.name().equalsIgnoreCase(
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION));
+ return UserGroupInformation.getLoginUser().isFromKeytab() &&
+ HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ZOOKEEPER_USE_KERBEROS);
} catch (IOException e) {
return false;
}
diff --git a/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java
index c486274..eb80cea 100644
--- a/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java
+++ b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.registry.impl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -43,24 +42,23 @@ public class TestZookeeperUtils {
}
@Test
- public void testHadoopKerberosZookeeperDefault() {
+ public void testHadoopAuthKerberosAndZookeeperUseKerberos() {
Mockito.when(ugi.isFromKeytab()).thenReturn(true);
+ Assert.assertTrue(HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ZOOKEEPER_USE_KERBEROS));
Assert.assertTrue(ZookeeperUtils.isKerberosEnabled(conf));
}
@Test
- public void testHadoopKerberosZookeeperSimple(){
+ public void testHadoopAuthKerberosAndZookeeperNoKerberos(){
Mockito.when(ugi.isFromKeytab()).thenReturn(true);
- conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname,
- AuthenticationMethod.SIMPLE.name());
+ conf.setBoolean(HiveConf.ConfVars.HIVE_ZOOKEEPER_USE_KERBEROS.varname, false);
Assert.assertFalse(ZookeeperUtils.isKerberosEnabled(conf));
}
@Test
- public void testHadoopSimpleZookeeperDefault(){
+ public void testHadoopAuthSimpleAndZookeeperKerberos(){
Mockito.when(ugi.isFromKeytab()).thenReturn(false);
- conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname,
- AuthenticationMethod.SIMPLE.name());
+ conf.setBoolean(HiveConf.ConfVars.HIVE_ZOOKEEPER_USE_KERBEROS.varname, false);
Assert.assertFalse(ZookeeperUtils.isKerberosEnabled(conf));
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index e2db0c7..baef0fe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -321,7 +321,7 @@ public class TezSessionState {
Credentials llapCredentials = null;
if (llapMode) {
- if (UserGroupInformation.isSecurityEnabled()) {
+ if (isKerberosEnabled(tezConfig)) {
llapCredentials = new Credentials();
llapCredentials.addToken(LlapTokenIdentifier.KIND_NAME, getLlapToken(user, tezConfig));
}
@@ -392,6 +392,10 @@ public class TezSessionState {
}
}
+ private boolean isKerberosEnabled(Configuration conf) {
+ return UserGroupInformation.isSecurityEnabled() && HiveConf.getBoolVar(conf, ConfVars.LLAP_USE_KERBEROS);
+ }
+
private static Token<LlapTokenIdentifier> getLlapToken(
String user, final Configuration conf) throws IOException {
// TODO: parts of this should be moved out of TezSession to reuse the clients, but there's
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 647df74..3f69ad8 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1046,6 +1046,10 @@ public class MetastoreConf {
"",
"Specifies which dynamic service discovery method to use. Currently we support only " +
"\"zookeeper\" to specify ZooKeeper based service discovery."),
+ THRIFT_ZOOKEEPER_USE_KERBEROS("metastore.zookeeper.kerberos.enabled",
+ "hive.zookeeper.kerberos.enabled", true,
+ "If ZooKeeper is configured for Kerberos authentication. This could be useful when cluster\n" +
+ "is kerberized, but Zookeeper is not."),
THRIFT_ZOOKEEPER_CLIENT_PORT("metastore.zookeeper.client.port",
"hive.metastore.zookeeper.client.port", "2181",
"The port of ZooKeeper servers to talk to.\n" +
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
index e8f2563..c9e85a6 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
@@ -31,9 +31,9 @@ import org.apache.curator.framework.api.ACLProvider;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
import org.apache.hadoop.security.token.delegation.MetastoreDelegationTokenSupport;
import org.apache.zookeeper.CreateMode;
@@ -85,7 +85,7 @@ public class ZooKeeperTokenStore implements DelegationTokenStore {
*/
private List<ACL> getDefaultAcl(Configuration conf) {
List<ACL> nodeAcls = new ArrayList<>();
- if (isZkSecurityEnabled(conf)) {
+ if (isKerberosEnabled(conf)) {
nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS));
} else {
nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE);
@@ -93,13 +93,10 @@ public class ZooKeeperTokenStore implements DelegationTokenStore {
return nodeAcls;
}
- /**
- * Check if ZooKeeper is configured with Kerberos authentication.
- */
- private boolean isZkSecurityEnabled(Configuration conf) {
+ private boolean isKerberosEnabled(Configuration conf) {
try {
- return UserGroupInformation.getLoginUser().isFromKeytab() && !AuthenticationMethod.SIMPLE.name().equalsIgnoreCase(
- getNonEmptyConfVar(conf, "hive.security.zookeeper.authentication"));
+ return UserGroupInformation.getLoginUser().isFromKeytab() &&
+ MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_USE_KERBEROS);
} catch (IOException e) {
return false;
}
@@ -135,7 +132,7 @@ public class ZooKeeperTokenStore implements DelegationTokenStore {
}
private void setupJAASConfig(Configuration conf) throws IOException {
- if (!isZkSecurityEnabled(conf)) {
+ if (!isKerberosEnabled(conf)) {
// The process has not logged in using keytab
// this should be a test mode, can't use keytab to authenticate
// with zookeeper.