You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/03/16 20:34:54 UTC
hive git commit: HIVE-18959 : Avoid creating extra pool of threads
within LLAP (Slim Bouguerra via Prasanth J)
Repository: hive
Updated Branches:
refs/heads/master f40d94476 -> 6218d8a2b
HIVE-18959 : Avoid creating extra pool of threads within LLAP (Slim Bouguerra via Prasanth J)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6218d8a2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6218d8a2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6218d8a2
Branch: refs/heads/master
Commit: 6218d8a2bd9b61014d81bf9b225a2891e0693401
Parents: f40d944
Author: Slim Bouguerra <sl...@gmail.com>
Authored: Fri Mar 16 13:34:02 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Mar 16 13:34:48 2018 -0700
----------------------------------------------------------------------
.../hive/druid/security/DruidKerberosUtil.java | 5 +-
.../hive/druid/security/KerberosHttpClient.java | 72 +++++++-------------
.../clientpositive/druid/druidmini_mv.q.out | 22 +++---
3 files changed, 39 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6218d8a2/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java
index e837925..84142af 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java
@@ -75,7 +75,7 @@ public class DruidKerberosUtil
byte[] outToken = gssContext.initSecContext(inToken, 0, inToken.length);
gssContext.dispose();
// Base64 encoded and stringified token for server
- log.info("Got valid challenge for host {}", serverName);
+ log.debug("Got valid challenge for host {}", serverName);
return new String(base64codec.encode(outToken), StandardCharsets.US_ASCII);
}
catch (GSSException | IllegalAccessException | NoSuchFieldException | ClassNotFoundException e) {
@@ -96,10 +96,11 @@ public class DruidKerberosUtil
boolean isSSL = uri.getScheme().equals("https");
List<HttpCookie> cookies = cookieStore.getCookies();
- for (HttpCookie c : cookies) {
+ for (int i = 0; i < cookies.size(); i++) {
// If this is a secured cookie and the current connection is non-secured,
// then, skip this cookie. We need to skip this cookie because, the cookie
// replay will not be transmitted to the server.
+ HttpCookie c = cookies.get(i);
if (c.getSecure() && !isSSL) {
continue;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6218d8a2/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java
index e7b3d99..2855bea 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java
@@ -19,15 +19,12 @@
package org.apache.hadoop.hive.druid.security;
import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.metamx.http.client.AbstractHttpClient;
import com.metamx.http.client.HttpClient;
import com.metamx.http.client.Request;
import com.metamx.http.client.response.HttpResponseHandler;
-import io.druid.concurrent.Execs;
import org.apache.hadoop.security.UserGroupInformation;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.joda.time.Duration;
@@ -39,7 +36,6 @@ import java.security.PrivilegedExceptionAction;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.Executor;
/**
* This is a slightly modified version of kerberos module borrowed from druid project
@@ -52,7 +48,6 @@ public class KerberosHttpClient extends AbstractHttpClient
protected static final org.slf4j.Logger log = LoggerFactory.getLogger(KerberosHttpClient.class);
private final HttpClient delegate;
private final CookieManager cookieManager;
- private final Executor exec = Execs.singleThreaded("KerberosHttpClient-%s");
public KerberosHttpClient(HttpClient delegate)
{
@@ -93,20 +88,15 @@ public class KerberosHttpClient extends AbstractHttpClient
if (DruidKerberosUtil.needToSendCredentials(cookieManager.getCookieStore(), uri)) {
// No Cookies for requested URI, authenticate user and add authentication header
- log.info(
- "No Auth Cookie found for URI{}. Existing Cookies{} Authenticating... ",
- uri,
- cookieManager.getCookieStore().getCookies()
+ log.debug("No Auth Cookie found for URI{}. Existing Cookies{} Authenticating... ", uri,
+ cookieManager.getCookieStore().getCookies()
);
// Assuming that a valid UGI with kerberos cred is created by HS2 or LLAP
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
currentUser.checkTGTAndReloginFromKeytab();
log.debug("The user credential is {}", currentUser);
- String challenge = currentUser.doAs(new PrivilegedExceptionAction<String>()
- {
- @Override
- public String run() throws Exception
- {
+ String challenge = currentUser.doAs(new PrivilegedExceptionAction<String>() {
+ @Override public String run() throws Exception {
return DruidKerberosUtil.kerberosChallenge(host);
}
});
@@ -116,44 +106,32 @@ public class KerberosHttpClient extends AbstractHttpClient
} else {
/* In this branch we had already a cookie that did expire
therefore we need to resend a valid Kerberos challenge*/
+ log.debug("Found Auth Cookie found for URI {} cookie {}", uri,
+ DruidKerberosUtil.getAuthCookie(cookieManager.getCookieStore(), uri).toString()
+ );
should_retry_on_unauthorized_response = true;
}
- ListenableFuture<RetryResponseHolder<Final>> internalFuture = delegate.go(
- request,
- new RetryIfUnauthorizedResponseHandler<Intermediate, Final>(new ResponseCookieHandler(
- request.getUrl().toURI(),
- cookieManager,
- httpResponseHandler
- )),
- duration
+ ListenableFuture<RetryResponseHolder<Final>> internalFuture = delegate.go(request, new RetryIfUnauthorizedResponseHandler<Intermediate, Final>(
+ new ResponseCookieHandler(request.getUrl().toURI(), cookieManager, httpResponseHandler
+ )), duration
);
- /* Handle response as Futures instead of inline code to avoid boiler plate repeated code. */
- Futures.addCallback(internalFuture, new FutureCallback<RetryResponseHolder<Final>>()
- {
- @Override
- public void onSuccess(RetryResponseHolder<Final> result)
- {
- if (should_retry_on_unauthorized_response && result.shouldRetry()) {
- log.info("Preparing for Retry");
- // remove Auth cookie
- DruidKerberosUtil.removeAuthCookie(cookieManager.getCookieStore(), uri);
- // clear existing cookie
- request.setHeader("Cookie", "");
- inner_go(request.copy(), httpResponseHandler, duration, future);
- } else {
- log.debug("Not retrying and returning future response");
- future.set(result.getObj());
- }
- }
-
- @Override
- public void onFailure(Throwable t)
- {
- future.setException(t);
- }
- }, exec);
+ RetryResponseHolder<Final> responseHolder = internalFuture.get();
+
+ if (should_retry_on_unauthorized_response && responseHolder.shouldRetry()) {
+ log.debug("Preparing for Retry boolean {} and result {}, object{} ",
+ should_retry_on_unauthorized_response, responseHolder.shouldRetry(), responseHolder.getObj()
+ );
+ // remove Auth cookie
+ DruidKerberosUtil.removeAuthCookie(cookieManager.getCookieStore(), uri);
+ // clear existing cookie
+ request.setHeader("Cookie", "");
+ inner_go(request.copy(), httpResponseHandler, duration, future);
+
+ } else {
+ future.set(responseHolder.getObj());
+ }
}
catch (Throwable e) {
throw Throwables.propagate(e);
http://git-wip-us.apache.org/repos/asf/hive/blob/6218d8a2/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
index 18ca78f..3ff7b39 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
@@ -330,7 +330,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable
- Statistics: Num rows: 42 Data size: 492 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 41 Data size: 480 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (a = 3) (type: boolean)
Statistics: Num rows: 5 Data size: 58 Basic stats: COMPLETE Column stats: NONE
@@ -346,7 +346,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable
- Statistics: Num rows: 42 Data size: 656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 41 Data size: 640 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((a = 3) and (d = 3)) (type: boolean)
Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE
@@ -451,34 +451,34 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable
- Statistics: Num rows: 42 Data size: 16072 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 41 Data size: 15680 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (a = 3) (type: boolean)
- Statistics: Num rows: 5 Data size: 1913 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1912 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: CAST( t AS timestamp with local time zone) (type: timestamp with local time zone), 3 (type: int), b (type: varchar(256)), c (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 1913 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1912 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), floor_hour(CAST( GenericUDFEpochMilli(_col0) AS TIMESTAMP)) (type: timestamp)
outputColumnNames: _col0, _col1, _col2, _col3, __time_granularity
- Statistics: Num rows: 5 Data size: 1913 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1912 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: __time_granularity (type: timestamp)
sort order: +
Map-reduce partition columns: __time_granularity (type: timestamp)
- Statistics: Num rows: 5 Data size: 1913 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1912 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double)
Reducer 2
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: int), VALUE._col2 (type: varchar(256)), VALUE._col3 (type: double), KEY.__time_granularity (type: timestamp)
outputColumnNames: _col0, _col1, _col2, _col3, __time_granularity
- Statistics: Num rows: 5 Data size: 1913 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1912 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Dp Sort State: PARTITION_SORTED
- Statistics: Num rows: 5 Data size: 1913 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1912 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
@@ -537,7 +537,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable
- Statistics: Num rows: 42 Data size: 492 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 41 Data size: 480 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (a = 3) (type: boolean)
Statistics: Num rows: 5 Data size: 58 Basic stats: COMPLETE Column stats: NONE
@@ -553,7 +553,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable
- Statistics: Num rows: 42 Data size: 656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 41 Data size: 640 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((a = 3) and (d = 3)) (type: boolean)
Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE