You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2016/03/04 19:49:16 UTC
hive git commit: HIVE-13169 : HiveServer2: Support delegation token
based connection when using http transport (Vaibhav Gumashtha, Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master 2c570dece -> 74facd79e
HIVE-13169 : HiveServer2: Support delegation token based connection when using http transport (Vaibhav Gumashtha, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/74facd79
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/74facd79
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/74facd79
Branch: refs/heads/master
Commit: 74facd79e2f10647a3da8efbec6c3165d899249f
Parents: 2c570de
Author: Thejas Nair <th...@hortonworks.com>
Authored: Fri Mar 4 10:49:08 2016 -0800
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Fri Mar 4 10:49:08 2016 -0800
----------------------------------------------------------------------
.../hive/thrift/TestHadoopAuthBridge23.java | 63 ++++---
.../hive/thrift/TestZooKeeperTokenStore.java | 12 +-
.../org/apache/hive/jdbc/HiveConnection.java | 11 +-
.../hive/jdbc/HttpTokenAuthInterceptor.java | 47 +++++
.../hadoop/hive/metastore/HiveMetaStore.java | 19 ++-
.../hive/service/auth/HiveAuthFactory.java | 76 ++++++---
.../org/apache/hive/service/cli/CLIService.java | 2 +-
.../service/cli/session/HiveSessionImpl.java | 2 +-
.../cli/session/HiveSessionImplwithUGI.java | 2 +-
.../cli/thrift/ThriftHttpCLIService.java | 2 +-
.../service/cli/thrift/ThriftHttpServlet.java | 26 ++-
.../thrift/DelegationTokenSecretManager.java | 25 +++
.../hive/thrift/HadoopThriftAuthBridge.java | 150 +---------------
.../hive/thrift/HiveDelegationTokenManager.java | 171 +++++++++++++++++++
.../TokenStoreDelegationTokenSecretManager.java | 10 --
.../hadoop/hive/thrift/ZooKeeperTokenStore.java | 16 +-
16 files changed, 392 insertions(+), 242 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
index 6d0776a..d07162b 100644
--- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
+++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.thrift;
import junit.framework.TestCase;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -27,6 +28,8 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.thrift.DelegationTokenStore.TokenStoreException;
+import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
@@ -65,6 +68,20 @@ public class TestHadoopAuthBridge23 extends TestCase {
*/
static volatile boolean isMetastoreTokenManagerInited;
+ public static class MyTokenStore extends MemoryTokenStore {
+ static volatile DelegationTokenStore TOKEN_STORE = null;
+ public void init(Object hmsHandler, ServerMode smode) throws TokenStoreException {
+ super.init(hmsHandler, smode);
+ TOKEN_STORE = this;
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ isMetastoreTokenManagerInited = true;
+ }
+ }
+
private static class MyHadoopThriftAuthBridge23 extends HadoopThriftAuthBridge23 {
@Override
public Server createServer(String keytabFile, String principalConf)
@@ -89,19 +106,7 @@ public class TestHadoopAuthBridge23 extends TestCase {
return new TUGIAssumingTransportFactory(transFactory, realUgi);
}
- static DelegationTokenStore TOKEN_STORE = new MemoryTokenStore();
-
- @Override
- protected DelegationTokenStore getTokenStore(Configuration conf) throws IOException {
- return TOKEN_STORE;
- }
- @Override
- public void startDelegationTokenSecretManager(Configuration conf, Object hms, ServerMode sm)
- throws IOException{
- super.startDelegationTokenSecretManager(conf, hms, sm);
- isMetastoreTokenManagerInited = true;
- }
}
}
@@ -142,6 +147,8 @@ public class TestHadoopAuthBridge23 extends TestCase {
"thrift://localhost:" + port);
System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(
System.getProperty("test.build.data", "/tmp")).toString());
+ System.setProperty(HiveConf.ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS.varname,
+ MyTokenStore.class.getName());
conf = new HiveConf(TestHadoopAuthBridge23.class);
MetaStoreUtils.startMetaStore(port, new MyHadoopThriftAuthBridge23());
}
@@ -155,7 +162,7 @@ public class TestHadoopAuthBridge23 extends TestCase {
TokenStoreDelegationTokenSecretManager tokenManager =
new TokenStoreDelegationTokenSecretManager(0, 60*60*1000, 60*60*1000, 0,
- MyHadoopThriftAuthBridge23.Server.TOKEN_STORE);
+ MyTokenStore.TOKEN_STORE);
// initializes current key
tokenManager.startThreads();
tokenManager.stopThreads();
@@ -171,31 +178,31 @@ public class TestHadoopAuthBridge23 extends TestCase {
assertTrue("Usernames don't match",
clientUgi.getShortUserName().equals(d.getUser().getShortUserName()));
- DelegationTokenInformation tokenInfo = MyHadoopThriftAuthBridge23.Server.TOKEN_STORE
+ DelegationTokenInformation tokenInfo = MyTokenStore.TOKEN_STORE
.getToken(d);
assertNotNull("token not in store", tokenInfo);
assertFalse("duplicate token add",
- MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.addToken(d, tokenInfo));
+ MyTokenStore.TOKEN_STORE.addToken(d, tokenInfo));
// check keys are copied from token store when token is loaded
TokenStoreDelegationTokenSecretManager anotherManager =
new TokenStoreDelegationTokenSecretManager(0, 0, 0, 0,
- MyHadoopThriftAuthBridge23.Server.TOKEN_STORE);
+ MyTokenStore.TOKEN_STORE);
assertEquals("master keys empty on init", 0,
anotherManager.getAllKeys().length);
assertNotNull("token loaded",
anotherManager.retrievePassword(d));
anotherManager.renewToken(t, clientUgi.getShortUserName());
assertEquals("master keys not loaded from store",
- MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.getMasterKeys().length,
+ MyTokenStore.TOKEN_STORE.getMasterKeys().length,
anotherManager.getAllKeys().length);
// cancel the delegation token
tokenManager.cancelDelegationToken(tokenStrForm);
assertNull("token not removed from store after cancel",
- MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.getToken(d));
+ MyTokenStore.TOKEN_STORE.getToken(d));
assertFalse("token removed (again)",
- MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.removeToken(d));
+ MyTokenStore.TOKEN_STORE.removeToken(d));
try {
anotherManager.retrievePassword(d);
fail("InvalidToken expected after cancel");
@@ -204,12 +211,12 @@ public class TestHadoopAuthBridge23 extends TestCase {
}
// token expiration
- MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.addToken(d,
+ MyTokenStore.TOKEN_STORE.addToken(d,
new DelegationTokenInformation(0, t.getPassword()));
- assertNotNull(MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.getToken(d));
+ assertNotNull(MyTokenStore.TOKEN_STORE.getToken(d));
anotherManager.removeExpiredTokens();
assertNull("Expired token not removed",
- MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.getToken(d));
+ MyTokenStore.TOKEN_STORE.getToken(d));
// key expiration - create an already expired key
anotherManager.startThreads(); // generates initial key
@@ -312,10 +319,9 @@ public class TestHadoopAuthBridge23 extends TestCase {
HadoopThriftAuthBridge.Server.authenticationMethod
.set(AuthenticationMethod.KERBEROS);
- HadoopThriftAuthBridge.Server.remoteAddress.set(InetAddress.getLocalHost());
return
HiveMetaStore.getDelegationToken(ownerUgi.getShortUserName(),
- realUgi.getShortUserName());
+ realUgi.getShortUserName(), InetAddress.getLocalHost().getHostAddress());
}
/**
@@ -368,15 +374,6 @@ public class TestHadoopAuthBridge23 extends TestCase {
//try out some metastore operations
createDBAndVerifyExistence(hiveClient);
- //check that getDelegationToken fails since we are not authenticating
- //over kerberos
- boolean pass = false;
- try {
- hiveClient.getDelegationToken(clientUgi.getUserName());
- } catch (MetaException ex) {
- pass = true;
- }
- assertTrue("Expected the getDelegationToken call to fail", pass == true);
hiveClient.close();
//Now cancel the delegation token
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
index 65a10e3..7800416 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
@@ -70,9 +70,9 @@ public class TestZooKeeperTokenStore extends TestCase {
private Configuration createConf(String zkPath) {
Configuration conf = new Configuration();
- conf.set(HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, "localhost:"
+ conf.set(HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, "localhost:"
+ this.zkPort);
- conf.set(HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE, zkPath);
+ conf.set(HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ZNODE, zkPath);
return conf;
}
@@ -80,7 +80,7 @@ public class TestZooKeeperTokenStore extends TestCase {
String ZK_PATH = "/zktokenstore-testTokenStorage";
ts = new ZooKeeperTokenStore();
Configuration conf = createConf(ZK_PATH);
- conf.set(HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_ACL, "world:anyone:cdrwa");
+ conf.set(HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ACL, "world:anyone:cdrwa");
ts.setConf(conf);
ts.init(null, ServerMode.METASTORE);
@@ -129,7 +129,7 @@ public class TestZooKeeperTokenStore extends TestCase {
String ZK_PATH = "/zktokenstore-testAclNoAuth";
Configuration conf = createConf(ZK_PATH);
conf.set(
- HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_ACL,
+ HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ACL,
"ip:127.0.0.1:r");
ts = new ZooKeeperTokenStore();
@@ -147,7 +147,7 @@ public class TestZooKeeperTokenStore extends TestCase {
String aclString = "sasl:hive/host@TEST.DOMAIN:cdrwa, fail-parse-ignored";
Configuration conf = createConf(ZK_PATH);
conf.set(
- HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_ACL,
+ HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ACL,
aclString);
List<ACL> aclList = ZooKeeperTokenStore.parseACLs(aclString);
@@ -167,7 +167,7 @@ public class TestZooKeeperTokenStore extends TestCase {
String ZK_PATH = "/zktokenstore-testAcl";
Configuration conf = createConf(ZK_PATH);
conf.set(
- HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_ACL,
+ HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ACL,
"ip:127.0.0.1:cdrwa,world:anyone:cdrwa");
ts = new ZooKeeperTokenStore();
ts.setConf(conf);
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index c3a17c1..873f421 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -296,8 +296,14 @@ public class HiveConnection implements java.sql.Connection {
new HttpKerberosRequestInterceptor(sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL),
host, getServerHttpUrl(useSsl), assumeSubject, cookieStore, cookieName, useSsl,
additionalHttpHeaders);
- }
- else {
+ } else {
+ // Check for delegation token, if present add it in the header
+ String tokenStr = getClientDelegationToken(sessConfMap);
+ if (tokenStr != null) {
+ requestInterceptor =
+ new HttpTokenAuthInterceptor(tokenStr, cookieStore, cookieName, useSsl,
+ additionalHttpHeaders);
+ } else {
/**
* Add an interceptor to pass username/password in the header.
* In https mode, the entire information is encrypted
@@ -305,6 +311,7 @@ public class HiveConnection implements java.sql.Connection {
requestInterceptor = new HttpBasicAuthInterceptor(getUserName(), getPassword(),
cookieStore, cookieName, useSsl,
additionalHttpHeaders);
+ }
}
// Configure http client for cookie based authentication
if (isCookieEnabled) {
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/jdbc/src/java/org/apache/hive/jdbc/HttpTokenAuthInterceptor.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HttpTokenAuthInterceptor.java b/jdbc/src/java/org/apache/hive/jdbc/HttpTokenAuthInterceptor.java
new file mode 100644
index 0000000..207ed9e
--- /dev/null
+++ b/jdbc/src/java/org/apache/hive/jdbc/HttpTokenAuthInterceptor.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.jdbc;
+
+import java.util.Map;
+
+import org.apache.http.HttpRequest;
+import org.apache.http.client.CookieStore;
+import org.apache.http.protocol.HttpContext;
+
+/**
+ * The class is instantiated with the username and password, it is then
+ * used to add header with these credentials to HTTP requests
+ *
+ */
+public class HttpTokenAuthInterceptor extends HttpRequestInterceptorBase {
+ private String tokenStr;
+ private static final String HIVE_DELEGATION_TOKEN_HEADER = "X-Hive-Delegation-Token";
+
+ public HttpTokenAuthInterceptor(String tokenStr, CookieStore cookieStore, String cn,
+ boolean isSSL, Map<String, String> additionalHeaders) {
+ super(cookieStore, cn, isSSL, additionalHeaders);
+ this.tokenStr = tokenStr;
+ }
+
+ @Override
+ protected void addHttpAuthHeader(HttpRequest httpRequest, HttpContext httpContext)
+ throws Exception {
+ httpRequest.addHeader(HIVE_DELEGATION_TOKEN_HEADER, tokenStr);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f425c6e..984e3fc 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -88,6 +88,7 @@ import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode;
import org.apache.hadoop.hive.thrift.TUGIContainingTransport;
+import org.apache.hadoop.hive.thrift.HiveDelegationTokenManager;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
@@ -178,6 +179,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
public static final String PUBLIC = "public";
private static HadoopThriftAuthBridge.Server saslServer;
+ private static HiveDelegationTokenManager delegationTokenManager;
private static boolean useSasl;
private static final class ChainedTTransportFactory extends TTransportFactory {
@@ -5240,7 +5242,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
try {
ret =
HiveMetaStore.getDelegationToken(token_owner,
- renewer_kerberos_principal_name);
+ renewer_kerberos_principal_name, getIpAddress());
} catch (IOException e) {
ex = e;
throw new MetaException(e.getMessage());
@@ -5979,7 +5981,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
*/
public static void cancelDelegationToken(String tokenStrForm
) throws IOException {
- saslServer.cancelDelegationToken(tokenStrForm);
+ delegationTokenManager.cancelDelegationToken(tokenStrForm);
}
/**
@@ -5988,9 +5990,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
* @param renewer
* the designated renewer
*/
- public static String getDelegationToken(String owner, String renewer)
+ public static String getDelegationToken(String owner, String renewer, String remoteAddr)
throws IOException, InterruptedException {
- return saslServer.getDelegationToken(owner, renewer);
+ return delegationTokenManager.getDelegationToken(owner, renewer, remoteAddr);
}
/**
@@ -6008,7 +6010,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
*/
public static long renewDelegationToken(String tokenStrForm
) throws IOException {
- return saslServer.renewDelegationToken(tokenStrForm);
+ return delegationTokenManager.renewDelegationToken(tokenStrForm);
}
/**
@@ -6224,8 +6226,11 @@ public class HiveMetaStore extends ThriftHiveMetastore {
saslServer = bridge.createServer(
conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE),
conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL));
- // start delegation token manager
- saslServer.startDelegationTokenSecretManager(conf, baseHandler, ServerMode.METASTORE);
+ // Start delegation token manager
+ delegationTokenManager = new HiveDelegationTokenManager();
+ delegationTokenManager.startDelegationTokenSecretManager(conf, baseHandler,
+ ServerMode.METASTORE);
+ saslServer.setSecretManager(delegationTokenManager.getSecretManager());
transFactory = saslServer.createTransportFactory(
MetaStoreUtils.getMetaStoreSaslProperties(conf));
processor = saslServer.wrapProcessor(
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
index 0c7455d..3d5e3a4 100644
--- a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
+++ b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
@@ -20,6 +20,7 @@ package org.apache.hive.service.auth;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -33,7 +34,6 @@ import javax.security.auth.login.LoginException;
import javax.security.sasl.AuthenticationException;
import javax.security.sasl.Sasl;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
@@ -42,11 +42,13 @@ import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.DBTokenStore;
+import org.apache.hadoop.hive.thrift.HiveDelegationTokenManager;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hive.service.ServiceException;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.thrift.ThriftCLIService;
import org.apache.thrift.TProcessorFactory;
@@ -93,6 +95,7 @@ public class HiveAuthFactory {
private final String transportMode;
private final HiveConf conf;
private String hadoopAuth;
+ private HiveDelegationTokenManager delegationTokenManager = null;
public static final String HS2_PROXY_USER = "hive.server2.proxy.user";
public static final String HS2_CLIENT_TOKEN = "hiveserver2ClientToken";
@@ -114,25 +117,28 @@ public class HiveAuthFactory {
authTypeStr = AuthTypes.NONE.getAuthName();
}
}
- if (hadoopAuth.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())
- && !authTypeStr.equalsIgnoreCase(AuthTypes.NOSASL.getAuthName())) {
- saslServer = ShimLoader.getHadoopThriftAuthBridge().createServer(
- conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB),
- conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL));
- // start delegation token manager
+ if (hadoopAuth.equalsIgnoreCase("kerberos")
+ && !authTypeStr.equalsIgnoreCase(AuthTypes.NOSASL.getAuthName())) {
+ saslServer =
+ ShimLoader.getHadoopThriftAuthBridge().createServer(
+ conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB),
+ conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL));
+
+ // Start delegation token manager
+ delegationTokenManager = new HiveDelegationTokenManager();
try {
- // rawStore is only necessary for DBTokenStore
+ // baseHandler is only necessary for DBTokenStore
HMSHandler baseHandler = null;
- String tokenStoreClass = conf.getVar(HiveConf.ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS);
-
+ String tokenStoreClass =
+ conf.getVar(HiveConf.ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS);
if (tokenStoreClass.equals(DBTokenStore.class.getName())) {
- baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf, true);
+ baseHandler = new HiveMetaStore.HMSHandler("New db based metastore server", conf, true);
}
-
- saslServer.startDelegationTokenSecretManager(conf, baseHandler, ServerMode.HIVESERVER2);
- }
- catch (MetaException|IOException e) {
- throw new TTransportException("Failed to start token manager", e);
+ delegationTokenManager.startDelegationTokenSecretManager(conf, baseHandler,
+ ServerMode.HIVESERVER2);
+ saslServer.setSecretManager(delegationTokenManager.getSecretManager());
+ } catch (MetaException | IOException e) {
+ throw new ServiceException("Failed to start token manager", e);
}
}
}
@@ -159,7 +165,7 @@ public class HiveAuthFactory {
}
if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) {
// no-op
- } else if (authTypeStr.equalsIgnoreCase(AuthTypes.NONE.getAuthName()) ||
+ } else if (authTypeStr.equalsIgnoreCase(AuthTypes.NONE.getAuthName()) ||
authTypeStr.equalsIgnoreCase(AuthTypes.LDAP.getAuthName()) ||
authTypeStr.equalsIgnoreCase(AuthTypes.PAM.getAuthName()) ||
authTypeStr.equalsIgnoreCase(AuthTypes.CUSTOM.getAuthName())) {
@@ -168,7 +174,7 @@ public class HiveAuthFactory {
authTypeStr, null, new HashMap<String, String>(),
new PlainSaslHelper.PlainServerCallbackHandler(authTypeStr));
} catch (AuthenticationException e) {
- throw new LoginException ("Error setting callback handler" + e);
+ throw new LoginException ("Error setting callback handler" + e);
}
} else {
throw new LoginException("Unsupported authentication type " + authTypeStr);
@@ -303,14 +309,16 @@ public class HiveAuthFactory {
}
// retrieve delegation token for the given user
- public String getDelegationToken(String owner, String renewer) throws HiveSQLException {
- if (saslServer == null) {
+ public String getDelegationToken(String owner, String renewer, String remoteAddr)
+ throws HiveSQLException {
+ if (delegationTokenManager == null) {
throw new HiveSQLException(
"Delegation token only supported over kerberos authentication", "08S01");
}
try {
- String tokenStr = saslServer.getDelegationTokenWithService(owner, renewer, HS2_CLIENT_TOKEN);
+ String tokenStr = delegationTokenManager.getDelegationTokenWithService(owner, renewer,
+ HS2_CLIENT_TOKEN, remoteAddr);
if (tokenStr == null || tokenStr.isEmpty()) {
throw new HiveSQLException(
"Received empty retrieving delegation token for user " + owner, "08S01");
@@ -326,12 +334,12 @@ public class HiveAuthFactory {
// cancel given delegation token
public void cancelDelegationToken(String delegationToken) throws HiveSQLException {
- if (saslServer == null) {
+ if (delegationTokenManager == null) {
throw new HiveSQLException(
"Delegation token only supported over kerberos authentication", "08S01");
}
try {
- saslServer.cancelDelegationToken(delegationToken);
+ delegationTokenManager.cancelDelegationToken(delegationToken);
} catch (IOException e) {
throw new HiveSQLException(
"Error canceling delegation token " + delegationToken, "08S01", e);
@@ -339,25 +347,39 @@ public class HiveAuthFactory {
}
public void renewDelegationToken(String delegationToken) throws HiveSQLException {
- if (saslServer == null) {
+ if (delegationTokenManager == null) {
throw new HiveSQLException(
"Delegation token only supported over kerberos authentication", "08S01");
}
try {
- saslServer.renewDelegationToken(delegationToken);
+ delegationTokenManager.renewDelegationToken(delegationToken);
} catch (IOException e) {
throw new HiveSQLException(
"Error renewing delegation token " + delegationToken, "08S01", e);
}
}
+ public String verifyDelegationToken(String delegationToken) throws HiveSQLException {
+ if (delegationTokenManager == null) {
+ throw new HiveSQLException(
+ "Delegation token only supported over kerberos authentication", "08S01");
+ }
+ try {
+ return delegationTokenManager.verifyDelegationToken(delegationToken);
+ } catch (IOException e) {
+ String msg = "Error verifying delegation token " + delegationToken;
+ LOG.error(msg, e);
+ throw new HiveSQLException(msg, "08S01", e);
+ }
+ }
+
public String getUserFromToken(String delegationToken) throws HiveSQLException {
- if (saslServer == null) {
+ if (delegationTokenManager == null) {
throw new HiveSQLException(
"Delegation token only supported over kerberos authentication", "08S01");
}
try {
- return saslServer.getUserFromToken(delegationToken);
+ return delegationTokenManager.getUserFromToken(delegationToken);
} catch (IOException e) {
throw new HiveSQLException(
"Error extracting user from delegation token " + delegationToken, "08S01", e);
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/service/src/java/org/apache/hive/service/cli/CLIService.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java b/service/src/java/org/apache/hive/service/cli/CLIService.java
index bba0090..ab30ae2 100644
--- a/service/src/java/org/apache/hive/service/cli/CLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/CLIService.java
@@ -481,7 +481,7 @@ public class CLIService extends CompositeService implements ICLIService {
String owner, String renewer) throws HiveSQLException {
String delegationToken = sessionManager.getSession(sessionHandle).
getDelegationToken(authFactory, owner, renewer);
- LOG.info(sessionHandle + ": getDelegationToken()");
+ LOG.info(sessionHandle + ": getDelegationToken()" + " owner: " + owner + ", renewer: " + renewer);
return delegationToken;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 1ddf0a4..2e45e2d 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -827,7 +827,7 @@ public class HiveSessionImpl implements HiveSession {
public String getDelegationToken(HiveAuthFactory authFactory, String owner, String renewer)
throws HiveSQLException {
HiveAuthFactory.verifyProxyAccess(getUserName(), owner, getIpAddress(), getHiveConf());
- return authFactory.getDelegationToken(owner, renewer);
+ return authFactory.getDelegationToken(owner, renewer, getIpAddress());
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
index ab4d12f..025b0b8 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
@@ -175,7 +175,7 @@ public class HiveSessionImplwithUGI extends HiveSessionImpl {
@Override
public String getDelegationToken(HiveAuthFactory authFactory, String owner,
String renewer) throws HiveSQLException {
- return authFactory.getDelegationToken(owner, renewer);
+ return authFactory.getDelegationToken(owner, renewer, getIpAddress());
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index cafe21f..2d9c8e2 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -123,7 +123,7 @@ public class ThriftHttpCLIService extends ThriftCLIService {
UserGroupInformation httpUGI = cliService.getHttpUGI();
String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION);
TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory, authType,
- serviceUGI, httpUGI);
+ serviceUGI, httpUGI, hiveAuthFactory);
// Context handler
final ServletContextHandler context = new ServletContextHandler(
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
index df16544..7e12fae 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
import org.apache.hive.service.CookieSigner;
import org.apache.hive.service.auth.AuthenticationProviderFactory;
import org.apache.hive.service.auth.AuthenticationProviderFactory.AuthMethods;
@@ -46,6 +47,7 @@ import org.apache.hive.service.auth.HiveAuthFactory;
import org.apache.hive.service.auth.HttpAuthUtils;
import org.apache.hive.service.auth.HttpAuthenticationException;
import org.apache.hive.service.auth.PasswdAuthenticationProvider;
+import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.session.SessionManager;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TProtocolFactory;
@@ -83,13 +85,17 @@ public class ThriftHttpServlet extends TServlet {
private int cookieMaxAge;
private boolean isCookieSecure;
private boolean isHttpOnlyCookie;
+ private final HiveAuthFactory hiveAuthFactory;
+ private static final String HIVE_DELEGATION_TOKEN_HEADER = "X-Hive-Delegation-Token";
public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory,
- String authType, UserGroupInformation serviceUGI, UserGroupInformation httpUGI) {
+ String authType, UserGroupInformation serviceUGI, UserGroupInformation httpUGI,
+ HiveAuthFactory hiveAuthFactory) {
super(processor, protocolFactory);
this.authType = authType;
this.serviceUGI = serviceUGI;
this.httpUGI = httpUGI;
+ this.hiveAuthFactory = hiveAuthFactory;
this.isCookieAuthEnabled = hiveConf.getBoolVar(
ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED);
// Initialize the cookie based authentication related variables.
@@ -132,7 +138,13 @@ public class ThriftHttpServlet extends TServlet {
if (clientUserName == null) {
// For a kerberos setup
if (isKerberosAuthMode(authType)) {
- clientUserName = doKerberosAuth(request);
+ String delegationToken = request.getHeader(HIVE_DELEGATION_TOKEN_HEADER);
+ // Each http request must have an Authorization header
+ if ((delegationToken != null) && (!delegationToken.isEmpty())) {
+ clientUserName = doTokenAuth(request, response);
+ } else {
+ clientUserName = doKerberosAuth(request);
+ }
}
// For password based authentication
else {
@@ -331,6 +343,16 @@ public class ThriftHttpServlet extends TServlet {
return userName;
}
+ private String doTokenAuth(HttpServletRequest request, HttpServletResponse response)
+ throws HttpAuthenticationException {
+ String tokenStr = request.getHeader(HIVE_DELEGATION_TOKEN_HEADER);
+ try {
+ return hiveAuthFactory.verifyDelegationToken(tokenStr);
+ } catch (HiveSQLException e) {
+ throw new HttpAuthenticationException(e);
+ }
+ }
+
/**
* Do the GSS-API kerberos authentication.
* We already have a logged in subject in the form of serviceUGI,
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java
index 19d1fbf..5299e18 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java
@@ -58,6 +58,31 @@ public class DelegationTokenSecretManager
return new DelegationTokenIdentifier();
}
+ /**
+ * Verify token string
+ * @param tokenStrForm
+ * @return user name
+ * @throws IOException
+ */
+ public synchronized String verifyDelegationToken(String tokenStrForm) throws IOException {
+ Token<DelegationTokenIdentifier> t = new Token<DelegationTokenIdentifier>();
+ t.decodeFromUrlString(tokenStrForm);
+
+ DelegationTokenIdentifier id = getTokenIdentifier(t);
+ verifyToken(id, t.getPassword());
+ return id.getUser().getShortUserName();
+ }
+
+ protected DelegationTokenIdentifier getTokenIdentifier(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ // turn bytes back into identifier for cache lookup
+ ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ DelegationTokenIdentifier id = createIdentifier();
+ id.readFields(in);
+ return id;
+ }
+
public synchronized void cancelDelegationToken(String tokenStrForm) throws IOException {
Token<DelegationTokenIdentifier> t= new Token<DelegationTokenIdentifier>();
t.decodeFromUrlString(tokenStrForm);
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
index d3b2ff5..8a4786c 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
@@ -39,25 +39,21 @@ import javax.security.sasl.SaslException;
import javax.security.sasl.SaslServer;
import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport;
+import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier;
+import org.apache.hadoop.hive.thrift.DelegationTokenSecretManager;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
-import org.apache.hadoop.security.authorize.AuthorizationException;
-import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.thrift.TException;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TProtocol;
@@ -289,38 +285,6 @@ public abstract class HadoopThriftAuthBridge {
public enum ServerMode {
HIVESERVER2, METASTORE
};
- public static final String DELEGATION_TOKEN_GC_INTERVAL =
- "hive.cluster.delegation.token.gc-interval";
- private final static long DELEGATION_TOKEN_GC_INTERVAL_DEFAULT = 3600000; // 1 hour
- //Delegation token related keys
- public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY =
- "hive.cluster.delegation.key.update-interval";
- public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT =
- 24*60*60*1000; // 1 day
- public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY =
- "hive.cluster.delegation.token.renew-interval";
- public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT =
- 24*60*60*1000; // 1 day
- public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY =
- "hive.cluster.delegation.token.max-lifetime";
- public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
- 7*24*60*60*1000; // 7 days
- public static final String DELEGATION_TOKEN_STORE_CLS =
- "hive.cluster.delegation.token.store.class";
- public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_STR =
- "hive.cluster.delegation.token.store.zookeeper.connectString";
- // alternate connect string specification configuration
- public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE =
- "hive.zookeeper.quorum";
-
- public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS =
- "hive.cluster.delegation.token.store.zookeeper.connectTimeoutMillis";
- public static final String DELEGATION_TOKEN_STORE_ZK_ZNODE =
- "hive.cluster.delegation.token.store.zookeeper.znode";
- public static final String DELEGATION_TOKEN_STORE_ZK_ACL =
- "hive.cluster.delegation.token.store.zookeeper.acl";
- public static final String DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT =
- "/hivedelegation";
protected final UserGroupInformation realUgi;
protected DelegationTokenSecretManager secretManager;
@@ -358,6 +322,10 @@ public abstract class HadoopThriftAuthBridge {
}
}
+ public void setSecretManager(DelegationTokenSecretManager secretManager) {
+ this.secretManager = secretManager;
+ }
+
/**
* Create a TTransportFactory that, upon connection of a client socket,
* negotiates a Kerberized SASL transport. The resulting TTransportFactory
@@ -377,7 +345,7 @@ public abstract class HadoopThriftAuthBridge {
/**
* Create a TSaslServerTransport.Factory that, upon connection of a client
- * socket, negotiates a Kerberized SASL transport.
+ * socket, negotiates a Kerberized SASL transport.
*
* @param saslProps Map of SASL properties
*/
@@ -430,109 +398,6 @@ public abstract class HadoopThriftAuthBridge {
return new TUGIAssumingProcessor(processor, secretManager, false);
}
- protected DelegationTokenStore getTokenStore(Configuration conf)
- throws IOException {
- String tokenStoreClassName = conf.get(DELEGATION_TOKEN_STORE_CLS, "");
- if (StringUtils.isBlank(tokenStoreClassName)) {
- return new MemoryTokenStore();
- }
- try {
- Class<? extends DelegationTokenStore> storeClass = Class
- .forName(tokenStoreClassName).asSubclass(
- DelegationTokenStore.class);
- return ReflectionUtils.newInstance(storeClass, conf);
- } catch (ClassNotFoundException e) {
- throw new IOException("Error initializing delegation token store: " + tokenStoreClassName,
- e);
- }
- }
-
-
- public void startDelegationTokenSecretManager(Configuration conf, Object hms, ServerMode smode)
- throws IOException {
- long secretKeyInterval =
- conf.getLong(DELEGATION_KEY_UPDATE_INTERVAL_KEY,
- DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
- long tokenMaxLifetime =
- conf.getLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY,
- DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
- long tokenRenewInterval =
- conf.getLong(DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
- DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
- long tokenGcInterval = conf.getLong(DELEGATION_TOKEN_GC_INTERVAL,
- DELEGATION_TOKEN_GC_INTERVAL_DEFAULT);
-
- DelegationTokenStore dts = getTokenStore(conf);
- dts.init(hms, smode);
- secretManager = new TokenStoreDelegationTokenSecretManager(secretKeyInterval,
- tokenMaxLifetime,
- tokenRenewInterval,
- tokenGcInterval, dts);
- secretManager.startThreads();
- }
-
-
- public String getDelegationToken(final String owner, final String renewer)
- throws IOException, InterruptedException {
- if (!authenticationMethod.get().equals(AuthenticationMethod.KERBEROS)) {
- throw new AuthorizationException(
- "Delegation Token can be issued only with kerberos authentication. " +
- "Current AuthenticationMethod: " + authenticationMethod.get()
- );
- }
- //if the user asking the token is same as the 'owner' then don't do
- //any proxy authorization checks. For cases like oozie, where it gets
- //a delegation token for another user, we need to make sure oozie is
- //authorized to get a delegation token.
- //Do all checks on short names
- UserGroupInformation currUser = UserGroupInformation.getCurrentUser();
- UserGroupInformation ownerUgi = UserGroupInformation.createRemoteUser(owner);
- if (!ownerUgi.getShortUserName().equals(currUser.getShortUserName())) {
- //in the case of proxy users, the getCurrentUser will return the
- //real user (for e.g. oozie) due to the doAs that happened just before the
- //server started executing the method getDelegationToken in the MetaStore
- ownerUgi = UserGroupInformation.createProxyUser(owner,
- UserGroupInformation.getCurrentUser());
- InetAddress remoteAddr = getRemoteAddress();
- ProxyUsers.authorize(ownerUgi,remoteAddr.getHostAddress(), null);
- }
- return ownerUgi.doAs(new PrivilegedExceptionAction<String>() {
-
- @Override
- public String run() throws IOException {
- return secretManager.getDelegationToken(renewer);
- }
- });
- }
-
-
- public String getDelegationTokenWithService(String owner, String renewer, String service)
- throws IOException, InterruptedException {
- String token = getDelegationToken(owner, renewer);
- return Utils.addServiceToToken(token, service);
- }
-
-
- public long renewDelegationToken(String tokenStrForm) throws IOException {
- if (!authenticationMethod.get().equals(AuthenticationMethod.KERBEROS)) {
- throw new AuthorizationException(
- "Delegation Token can be issued only with kerberos authentication. " +
- "Current AuthenticationMethod: " + authenticationMethod.get()
- );
- }
- return secretManager.renewDelegationToken(tokenStrForm);
- }
-
-
- public String getUserFromToken(String tokenStr) throws IOException {
- return secretManager.getUserFromToken(tokenStr);
- }
-
-
- public void cancelDelegationToken(String tokenStrForm) throws IOException {
- secretManager.cancelDelegationToken(tokenStrForm);
- }
-
final static ThreadLocal<InetAddress> remoteAddress =
new ThreadLocal<InetAddress>() {
@@ -542,7 +407,6 @@ public abstract class HadoopThriftAuthBridge {
}
};
-
public InetAddress getRemoteAddress() {
return remoteAddress.get();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java
new file mode 100644
index 0000000..9ecb0ee
--- /dev/null
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.hive.thrift;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.shims.Utils;
+import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.util.ReflectionUtils;
+
+public class HiveDelegationTokenManager {
+
+ public static final String DELEGATION_TOKEN_GC_INTERVAL =
+ "hive.cluster.delegation.token.gc-interval";
+ private final static long DELEGATION_TOKEN_GC_INTERVAL_DEFAULT = 3600000; // 1 hour
+ // Delegation token related keys
+ public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY =
+ "hive.cluster.delegation.key.update-interval";
+ public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT =
+ 24*60*60*1000; // 1 day
+ public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY =
+ "hive.cluster.delegation.token.renew-interval";
+ public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT =
+ 24*60*60*1000; // 1 day
+ public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY =
+ "hive.cluster.delegation.token.max-lifetime";
+ public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
+ 7*24*60*60*1000; // 7 days
+ public static final String DELEGATION_TOKEN_STORE_CLS =
+ "hive.cluster.delegation.token.store.class";
+ public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_STR =
+ "hive.cluster.delegation.token.store.zookeeper.connectString";
+ // Alternate connect string specification configuration
+ public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE =
+ "hive.zookeeper.quorum";
+
+ public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS =
+ "hive.cluster.delegation.token.store.zookeeper.connectTimeoutMillis";
+ public static final String DELEGATION_TOKEN_STORE_ZK_ZNODE =
+ "hive.cluster.delegation.token.store.zookeeper.znode";
+ public static final String DELEGATION_TOKEN_STORE_ZK_ACL =
+ "hive.cluster.delegation.token.store.zookeeper.acl";
+ public static final String DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT =
+ "/hivedelegation";
+
+ protected DelegationTokenSecretManager secretManager;
+
+ public HiveDelegationTokenManager() {
+ }
+
+ public DelegationTokenSecretManager getSecretManager() {
+ return secretManager;
+ }
+
+ public void startDelegationTokenSecretManager(Configuration conf, Object hms, ServerMode smode)
+ throws IOException {
+ long secretKeyInterval =
+ conf.getLong(DELEGATION_KEY_UPDATE_INTERVAL_KEY, DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
+ long tokenMaxLifetime =
+ conf.getLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+ long tokenRenewInterval =
+ conf.getLong(DELEGATION_TOKEN_RENEW_INTERVAL_KEY, DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+ long tokenGcInterval =
+ conf.getLong(DELEGATION_TOKEN_GC_INTERVAL, DELEGATION_TOKEN_GC_INTERVAL_DEFAULT);
+
+ DelegationTokenStore dts = getTokenStore(conf);
+ dts.init(hms, smode);
+ secretManager =
+ new TokenStoreDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime,
+ tokenRenewInterval, tokenGcInterval, dts);
+ secretManager.startThreads();
+ }
+
+ public String getDelegationToken(final String owner, final String renewer, String remoteAddr)
+ throws IOException,
+ InterruptedException {
+ /**
+ * If the user asking the token is same as the 'owner' then don't do
+ * any proxy authorization checks. For cases like oozie, where it gets
+ * a delegation token for another user, we need to make sure oozie is
+ * authorized to get a delegation token.
+ */
+ // Do all checks on short names
+ UserGroupInformation currUser = UserGroupInformation.getCurrentUser();
+ UserGroupInformation ownerUgi = UserGroupInformation.createRemoteUser(owner);
+ if (!ownerUgi.getShortUserName().equals(currUser.getShortUserName())) {
+ // in the case of proxy users, the getCurrentUser will return the
+ // real user (for e.g. oozie) due to the doAs that happened just before the
+ // server started executing the method getDelegationToken in the MetaStore
+ ownerUgi = UserGroupInformation.createProxyUser(owner, UserGroupInformation.getCurrentUser());
+ ProxyUsers.authorize(ownerUgi, remoteAddr, null);
+ }
+ return ownerUgi.doAs(new PrivilegedExceptionAction<String>() {
+
+ @Override
+ public String run() throws IOException {
+ return secretManager.getDelegationToken(renewer);
+ }
+ });
+ }
+
+ public String getDelegationTokenWithService(String owner, String renewer, String service, String remoteAddr)
+ throws IOException, InterruptedException {
+ String token = getDelegationToken(owner, renewer, remoteAddr);
+ return Utils.addServiceToToken(token, service);
+ }
+
+ public long renewDelegationToken(String tokenStrForm)
+ throws IOException {
+ return secretManager.renewDelegationToken(tokenStrForm);
+ }
+
+ public String getUserFromToken(String tokenStr) throws IOException {
+ return secretManager.getUserFromToken(tokenStr);
+ }
+
+ public void cancelDelegationToken(String tokenStrForm) throws IOException {
+ secretManager.cancelDelegationToken(tokenStrForm);
+ }
+
+ /**
+ * Verify token string
+ * @param tokenStrForm
+ * @return user name
+ * @throws IOException
+ */
+ public String verifyDelegationToken(String tokenStrForm) throws IOException {
+ return secretManager.verifyDelegationToken(tokenStrForm);
+ }
+
+ private DelegationTokenStore getTokenStore(Configuration conf) throws IOException {
+ String tokenStoreClassName = conf.get(DELEGATION_TOKEN_STORE_CLS, "");
+ if (StringUtils.isBlank(tokenStoreClassName)) {
+ return new MemoryTokenStore();
+ }
+ try {
+ Class<? extends DelegationTokenStore> storeClass =
+ Class.forName(tokenStoreClassName).asSubclass(DelegationTokenStore.class);
+ return ReflectionUtils.newInstance(storeClass, conf);
+ } catch (ClassNotFoundException e) {
+ throw new IOException("Error initializing delegation token store: " + tokenStoreClassName, e);
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java
index 87b418e..abe8cc2 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java
@@ -76,16 +76,6 @@ public class TokenStoreDelegationTokenSecretManager extends DelegationTokenSecre
this.tokenStore = sharedStore;
}
- protected DelegationTokenIdentifier getTokenIdentifier(Token<DelegationTokenIdentifier> token)
- throws IOException {
- // turn bytes back into identifier for cache lookup
- ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
- DataInputStream in = new DataInputStream(buf);
- DelegationTokenIdentifier id = createIdentifier();
- id.readFields(in);
- return id;
- }
-
protected Map<Integer, DelegationKey> reloadKeys() {
// read keys from token store
String[] allKeys = tokenStore.getMasterKeys();
http://git-wip-us.apache.org/repos/asf/hive/blob/74facd79/shims/common/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
index 528e55d..885ec56 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
@@ -436,32 +436,32 @@ public class ZooKeeperTokenStore implements DelegationTokenStore {
public void init(Object hmsHandler, ServerMode smode) {
this.serverMode = smode;
zkConnectString =
- conf.get(HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, null);
+ conf.get(HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, null);
if (zkConnectString == null || zkConnectString.trim().isEmpty()) {
// try alternate config param
zkConnectString =
conf.get(
- HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE,
+ HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE,
null);
if (zkConnectString == null || zkConnectString.trim().isEmpty()) {
throw new IllegalArgumentException("Zookeeper connect string has to be specifed through "
- + "either " + HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR
+ + "either " + HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR
+ " or "
- + HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE
+ + HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE
+ WHEN_ZK_DSTORE_MSG);
}
}
connectTimeoutMillis =
conf.getInt(
- HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS,
+ HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS,
CuratorFrameworkFactory.builder().getConnectionTimeoutMs());
- String aclStr = conf.get(HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_ACL, null);
+ String aclStr = conf.get(HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ACL, null);
if (StringUtils.isNotBlank(aclStr)) {
this.newNodeAcl = parseACLs(aclStr);
}
rootNode =
- conf.get(HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE,
- HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT) + serverMode;
+ conf.get(HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ZNODE,
+ HiveDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT) + serverMode;
try {
// Install the JAAS Configuration for the runtime