You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2023/11/20 12:13:21 UTC
(hive) branch branch-3 updated: HIVE-27888: Backport of HIVE-22429, HIVE-14898, HIVE-22231, HIVE-20507, HIVE-24786 to branch-3 (#4878)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new b7316374cb3 HIVE-27888: Backport of HIVE-22429, HIVE-14898, HIVE-22231, HIVE-20507, HIVE-24786 to branch-3 (#4878)
b7316374cb3 is described below
commit b7316374cb35988ebb4bed3c96262b85bba22fc2
Author: Aman Raj <10...@users.noreply.github.com>
AuthorDate: Mon Nov 20 17:43:13 2023 +0530
HIVE-27888: Backport of HIVE-22429, HIVE-14898, HIVE-22231, HIVE-20507, HIVE-24786 to branch-3 (#4878)
* HIVE-22429: Migrated clustered tables using bucketing_version 1 on hive 3 uses bucketing_version 2 for inserts (Ramesh Kumar Thangarajan, reviewed by Jesus Camacho Rodriguez)
* HIVE-14898: HS2 shouldn't log callstack for an empty auth header error
* HIVE-22231: Hive query with big size via knox fails with Broken pipe Write failed (Denys Kuzmenko via Peter Vary)
* HIVE-20507: Beeline: Add a utility command to retrieve all uris from beeline-site.xml
* HIVE-24786: JDBC HttpClient should retry for idempotent and unsent http methods (#1983)
* HIVE-24786: JDBC HttpClient should retry for idempotent and unsent http methods
---------
Co-authored-by: Ramesh Kumar Thangarajan <ra...@cloudera.com>
Co-authored-by: Daniel Dai <da...@gmail.com>
Co-authored-by: denys kuzmenko <dk...@cloudera.com>
Co-authored-by: Vaibhav Gumashta <vg...@hortonworks.com>
Co-authored-by: Prasanth Jayachandran <pr...@apache.org>
Co-authored-by: Prasanth Jayachandran <pj...@cloudera.com>
---------
Signed-off-by: Sankar Hariappan <sa...@apache.org>
Closes (#4878)
---
.../src/java/org/apache/hive/beeline/BeeLine.java | 57 ++++++-
.../java/org/apache/hive/beeline/BeeLineOpts.java | 10 ++
beeline/src/main/resources/BeeLine.properties | 1 +
.../java/org/apache/hive/jdbc/HiveConnection.java | 186 ++++++++++++++++++++-
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 2 +-
.../apache/hadoop/hive/ql/parse/TezCompiler.java | 3 -
.../ldap/HttpEmptyAuthenticationException.java | 23 +++
.../hive/service/cli/thrift/ThriftHttpServlet.java | 22 ++-
8 files changed, 291 insertions(+), 13 deletions(-)
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index 73653d4217e..01adb1e1ff5 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -65,6 +65,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.ResourceBundle;
import java.util.ServiceLoader;
@@ -94,6 +95,7 @@ import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils;
import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser;
import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser;
import org.apache.hive.common.util.ShutdownHookManager;
+import org.apache.hive.jdbc.HiveConnection;
import org.apache.hive.jdbc.JdbcUriParseException;
import org.apache.hive.jdbc.Utils;
import org.apache.hive.jdbc.Utils.JdbcConnectionParams;
@@ -389,6 +391,12 @@ public class BeeLine implements Closeable {
.withLongOpt("help")
.withDescription("Display this message")
.create('h'));
+
+ // -getUrlsFromBeelineSite
+ options.addOption(OptionBuilder
+ .withLongOpt("getUrlsFromBeelineSite")
+ .withDescription("Print all urls from beeline-site.xml, if it is present in the classpath")
+ .create());
// Substitution option --hivevar
options.addOption(OptionBuilder
@@ -712,7 +720,7 @@ public class BeeLine implements Closeable {
private boolean isBeeLineOpt(String arg) {
return arg.startsWith("--") && !(HIVE_VAR_PREFIX.equals(arg) || (HIVE_CONF_PREFIX.equals(arg))
- || "--help".equals(arg) || PROP_FILE_PREFIX.equals(arg));
+ || "--help".equals(arg) || PROP_FILE_PREFIX.equals(arg) || "--getUrlsFromBeelineSite".equals(arg));
}
}
@@ -843,6 +851,12 @@ public class BeeLine implements Closeable {
getOpts().setHelpAsked(true);
return true;
}
+
+ if (cl.hasOption("getUrlsFromBeelineSite")) {
+ printBeelineSiteUrls();
+ getOpts().setBeelineSiteUrlsAsked(true);
+ return true;
+ }
Properties hiveVars = cl.getOptionProperties("hivevar");
for (String key : hiveVars.stringPropertyNames()) {
@@ -919,6 +933,44 @@ public class BeeLine implements Closeable {
return false;
}
+ private void printBeelineSiteUrls() {
+ BeelineSiteParser beelineSiteParser = getUserBeelineSiteParser();
+ if (!beelineSiteParser.configExists()) {
+ output("No beeline-site.xml in the path", true);
+ }
+ if (beelineSiteParser.configExists()) {
+ // Get the named url from user specific config file if present
+ try {
+ Properties userNamedConnectionURLs = beelineSiteParser.getConnectionProperties();
+ userNamedConnectionURLs.remove(BeelineSiteParser.DEFAULT_NAMED_JDBC_URL_PROPERTY_KEY);
+ StringBuilder sb = new StringBuilder("urls: ");
+ for (Entry<Object, Object> entry : userNamedConnectionURLs.entrySet()) {
+ String urlFromBeelineSite = (String) entry.getValue();
+ if (isZkBasedUrl(urlFromBeelineSite)) {
+ List<String> jdbcUrls = HiveConnection.getAllUrlStrings(urlFromBeelineSite);
+ for (String jdbcUrl : jdbcUrls) {
+ sb.append(jdbcUrl + ", ");
+ }
+ } else {
+ sb.append(urlFromBeelineSite + ", ");
+ }
+ }
+ output(sb.toString(), true);
+ } catch (Exception e) {
+ output(e.getMessage(), true);
+ return;
+ }
+ }
+ }
+
+ private boolean isZkBasedUrl(String urlFromBeelineSite) {
+ String zkJdbcUriParam = ("serviceDiscoveryMode=zooKeeper").toLowerCase();
+ if (urlFromBeelineSite.toLowerCase().contains(zkJdbcUriParam)) {
+ return true;
+ }
+ return false;
+ }
+
private void setHiveConfVar(String key, String val) {
getOpts().getHiveConfVariables().put(key, val);
if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) {
@@ -1060,6 +1112,9 @@ public class BeeLine implements Closeable {
if (getOpts().isHelpAsked()) {
return 0;
}
+ if (getOpts().isBeelineSiteUrlsAsked()) {
+ return 0;
+ }
if (getOpts().getScriptFile() != null) {
return executeFile(getOpts().getScriptFile());
}
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
index 5967b4d7bc0..de1cf092df2 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
@@ -115,6 +115,7 @@ public class BeeLineOpts implements Completer {
private Map<String, String> hiveVariables = new HashMap<String, String>();
private Map<String, String> hiveConfVariables = new HashMap<String, String>();
private boolean helpAsked;
+ private boolean beelineSiteUrlsAsked;
private String lastConnectedUrl = null;
@@ -687,7 +688,16 @@ public class BeeLineOpts implements Completer {
public boolean isHelpAsked() {
return helpAsked;
}
+
+ public void setBeelineSiteUrlsAsked(boolean beelineSiteUrlsAsked) {
+ this.beelineSiteUrlsAsked = beelineSiteUrlsAsked;
+ }
+
+ public boolean isBeelineSiteUrlsAsked() {
+ return beelineSiteUrlsAsked;
+ }
+
public String getLastConnectedUrl(){
return lastConnectedUrl;
}
diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties
index a4e342d089b..e3b0ba38303 100644
--- a/beeline/src/main/resources/BeeLine.properties
+++ b/beeline/src/main/resources/BeeLine.properties
@@ -212,6 +212,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \
\ --delimiter=DELIMITER set the query delimiter; multi-char delimiters are allowed, but quotation\n \
\ marks, slashes, and -- are not allowed; defaults to ;\n \
\ --convertBinaryArrayToString=[true/false] display binary column data as string or as byte array \n \
+\ --getUrlsFromBeelineSite Print all urls from beeline-site.xml, if it is present in the classpath\n \
\ --help display this message\n \
\n \
\ Example:\n \
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index 9b364f9c90b..70696281b6a 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -43,11 +43,17 @@ import org.apache.hive.service.rpc.thrift.TProtocolVersion;
import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenReq;
import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenResp;
import org.apache.hive.service.rpc.thrift.TSessionHandle;
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
+import org.apache.http.NoHttpResponseException;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.ServiceUnavailableRetryStrategy;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
@@ -57,9 +63,11 @@ import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.RequestWrapper;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContexts;
+import org.apache.http.util.Args;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.THttpClient;
@@ -69,6 +77,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
import javax.net.ssl.TrustManagerFactory;
import javax.security.auth.Subject;
import javax.security.sasl.Sasl;
@@ -76,12 +85,17 @@ import javax.security.sasl.SaslException;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
+import java.io.InterruptedIOException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
import java.security.AccessControlContext;
import java.security.AccessController;
import java.security.KeyStore;
@@ -105,6 +119,7 @@ import java.sql.Savepoint;
import java.sql.Statement;
import java.sql.Struct;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@@ -144,6 +159,7 @@ public class HiveConnection implements java.sql.Connection {
private String wmPool = null, wmApp = null;
private Properties clientInfo;
private Subject loggedInSubject;
+ private int maxRetries = 1;
/**
* Get all direct HiveServer2 URLs from a ZooKeeper based HiveServer2 URL
@@ -162,6 +178,79 @@ public class HiveConnection implements java.sql.Connection {
return ZooKeeperHiveClientHelper.getDirectParamsList(params);
}
+ public static List<String> getAllUrlStrings(String zookeeperBasedHS2Url) throws Exception {
+ List<String> jdbcUrls = new ArrayList<>();
+ List<JdbcConnectionParams> allConnectionParams = getAllUrls(zookeeperBasedHS2Url);
+ for (JdbcConnectionParams cp : allConnectionParams) {
+ String jdbcUrl = makeDirectJDBCUrlFromConnectionParams(cp);
+ if ((jdbcUrl != null) && (!jdbcUrl.isEmpty())) {
+ jdbcUrls.add(jdbcUrl);
+ }
+ }
+ return jdbcUrls;
+ }
+
+ private static String makeDirectJDBCUrlFromConnectionParams(JdbcConnectionParams cp) {
+ // Direct JDBC Url format:
+ // jdbc:hive2://<host1>:<port1>/dbName;sess_var_list?hive_conf_list#hive_var_list
+ StringBuilder url = new StringBuilder("");
+ if (cp != null) {
+ if (cp.getHost() != null) {
+ url.append(cp.getHost());
+ url.append(":");
+ url.append(cp.getPort());
+ url.append("/");
+ url.append(cp.getDbName());
+ // Add session vars
+ if ((cp.getSessionVars() != null) && (!cp.getSessionVars().isEmpty())) {
+ for (Entry<String, String> sessVar : cp.getSessionVars().entrySet()) {
+ if ((sessVar.getKey().equalsIgnoreCase(JdbcConnectionParams.SERVICE_DISCOVERY_MODE))
+ || (sessVar.getKey().equalsIgnoreCase(JdbcConnectionParams.ZOOKEEPER_NAMESPACE))) {
+ continue;
+ }
+ url.append(";");
+ url.append(sessVar.getKey());
+ url.append("=");
+ url.append(sessVar.getValue());
+ }
+ }
+ // Add hive confs
+ if ((cp.getHiveConfs() != null) && (!cp.getHiveConfs().isEmpty())) {
+ url.append("?");
+ boolean firstKV = true;
+ for (Entry<String, String> hiveConf : cp.getHiveConfs().entrySet()) {
+ if (!firstKV) {
+ url.append(";");
+ } else {
+ firstKV = false;
+ }
+ url.append(hiveConf.getKey());
+ url.append("=");
+ url.append(hiveConf.getValue());
+ }
+ }
+ // Add hive vars
+ if ((cp.getHiveVars() != null) && (!cp.getHiveVars().isEmpty())) {
+ url.append("#");
+ boolean firstKV = true;
+ for (Entry<String, String> hiveVar : cp.getHiveVars().entrySet()) {
+ if (!firstKV) {
+ url.append(";");
+ } else {
+ firstKV = false;
+ }
+ url.append(hiveVar.getKey());
+ url.append("=");
+ url.append(hiveVar.getValue());
+ }
+ }
+ } else {
+ return url.toString();
+ }
+ }
+ return url.toString();
+ }
+
public HiveConnection(String uri, Properties info) throws SQLException {
try {
connParams = Utils.parseURL(uri, info);
@@ -488,21 +577,99 @@ public class HiveConnection implements java.sql.Connection {
} else {
httpClientBuilder = HttpClientBuilder.create();
}
- // In case the server's idletimeout is set to a lower value, it might close it's side of
- // connection. However we retry one more time on NoHttpResponseException
+
+ // Beeline <------> LB <------> Reverse Proxy <-----> Hiveserver2
+ // In case of deployments like above, the LoadBalancer (LB) can be configured with Idle Timeout after which the LB
+ // will send TCP RST to Client (Beeline) and Backend (Reverse Proxy). If user is connected to beeline, idle for
+ // sometime and resubmits a query after the idle timeout there is a broken pipe between beeline and LB. When Beeline
+ // tries to submit the query one of two things happen, it either hangs or times out (if socketTimeout is defined in
+ // the jdbc param). The hang is because of the default infinite socket timeout for which there is no auto-recovery
+ // (user have to manually interrupt the query). If the socketTimeout jdbc param was specified, beeline will receive
+ // SocketTimeoutException (Read Timeout) or NoHttpResponseException both of which can be retried if maxRetries is
+ // also specified by the user (jdbc param).
+ // The following retry handler handles the above cases in addition to retries for idempotent and unsent requests.
httpClientBuilder.setRetryHandler(new HttpRequestRetryHandler() {
+ // This handler is mostly a copy of DefaultHttpRequestRetryHandler except it also retries some exceptions
+ // which could be thrown in certain cases where idle timeout from intermediate proxy triggers a connection reset.
+ private final List<Class<? extends IOException>> nonRetriableClasses = Arrays.asList(
+ InterruptedIOException.class,
+ UnknownHostException.class,
+ ConnectException.class,
+ SSLException.class);
+ // socket exceptions could happen because of timeout, broken pipe or server not responding in which case it is
+ // better to reopen the connection and retry if user specified maxRetries
+ private final List<Class<? extends IOException>> retriableClasses = Arrays.asList(
+ SocketTimeoutException.class,
+ SocketException.class,
+ NoHttpResponseException.class
+ );
+
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
- if (executionCount > 1) {
- LOG.info("Retry attempts to connect to server exceeded.");
+ Args.notNull(exception, "Exception parameter");
+ Args.notNull(context, "HTTP context");
+ if (executionCount > maxRetries) {
+ // Do not retry if over max retry count
+ LOG.error("Max retries (" + maxRetries + ") exhausted.", exception);
+ return false;
+ }
+ if (this.retriableClasses.contains(exception.getClass())) {
+ LOG.info("Retrying " + exception.getClass() + " as it is in retriable classes list.");
+ return true;
+ }
+ if (this.nonRetriableClasses.contains(exception.getClass())) {
+ LOG.info("Not retrying as the class (" + exception.getClass() + ") is non-retriable class.");
return false;
+ } else {
+ for (final Class<? extends IOException> rejectException : this.nonRetriableClasses) {
+ if (rejectException.isInstance(exception)) {
+ LOG.info("Not retrying as the class (" + exception.getClass() + ") is an instance of is non-retriable class.");;
+ return false;
+ }
+ }
}
- if (exception instanceof org.apache.http.NoHttpResponseException) {
- LOG.info("Could not connect to the server. Retrying one more time.");
+ final HttpClientContext clientContext = HttpClientContext.adapt(context);
+ final HttpRequest request = clientContext.getRequest();
+
+ if(requestIsAborted(request)){
+ LOG.info("Not retrying as request is aborted.");
+ return false;
+ }
+
+ if (handleAsIdempotent(request)) {
+ LOG.info("Retrying idempotent request. Attempt " + executionCount + " of " + maxRetries);
+ // Retry if the request is considered idempotent
return true;
}
+
+ if (!clientContext.isRequestSent()) {
+ LOG.info("Retrying unsent request. Attempt " + executionCount + " of " + maxRetries);
+ // Retry if the request has not been sent fully or
+ // if it's OK to retry methods that have been sent
+ return true;
+ }
+
+ LOG.info("Not retrying as the request is not idempotent or is already sent.");
+ // otherwise do not retry
return false;
}
+
+ // requests that handles "Expect continue" handshakes. If server received the header and is waiting for body
+ // then those requests can be retried. Most basic http method methods except DELETE are idempotent as long as they
+ // are not aborted.
+ protected boolean handleAsIdempotent(final HttpRequest request) {
+ return !(request instanceof HttpEntityEnclosingRequest);
+ }
+
+ // checks if the request got aborted
+ protected boolean requestIsAborted(final HttpRequest request) {
+ HttpRequest req = request;
+ if (request instanceof RequestWrapper) { // does not forward request to original
+ req = ((RequestWrapper) request).getOriginal();
+ }
+ return (req instanceof HttpUriRequest && ((HttpUriRequest)req).isAborted());
+ }
+
});
// Add the request interceptor to the client builder
@@ -511,6 +678,13 @@ public class HiveConnection implements java.sql.Connection {
// Add an interceptor to add in an XSRF header
httpClientBuilder.addInterceptorLast(new XsrfHttpRequestInterceptor());
+ // set the specified timeout (socketTimeout jdbc param) for http connection as well
+ RequestConfig config = RequestConfig.custom()
+ .setConnectTimeout(loginTimeout * 1000)
+ .setConnectionRequestTimeout(loginTimeout * 1000)
+ .setSocketTimeout(loginTimeout * 1000).build();
+ httpClientBuilder.setDefaultRequestConfig(config);
+
// Configure http client for SSL
if (useSsl) {
String useTwoWaySSL = sessConfMap.get(JdbcConnectionParams.USE_TWO_WAY_SSL);
diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
index 324776d3d5a..e54fdcebfaa 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
@@ -115,7 +115,7 @@ public class Utils {
// Use ZooKeeper for indirection while using dynamic service discovery
public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER = "zooKeeper";
public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER_HA = "zooKeeperHA";
- static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace";
+ public static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace";
// Default namespace value on ZooKeeper.
// This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri.
static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2";
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index 95ef33ffe20..c172c5d1baf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -1660,9 +1660,6 @@ public class TezCompiler extends TaskCompiler {
for (FileSinkOperator fsOp : fsOpsAll) {
- if (!fsOp.getConf().getTableInfo().isSetBucketingVersion()) {
- continue;
- }
// Look for direct parent ReduceSinkOp
// If there are more than 1 parent, bail out.
Operator<?> parent = fsOp;
diff --git a/service/src/java/org/apache/hive/service/auth/ldap/HttpEmptyAuthenticationException.java b/service/src/java/org/apache/hive/service/auth/ldap/HttpEmptyAuthenticationException.java
new file mode 100644
index 00000000000..b6b71bcc7ad
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/auth/ldap/HttpEmptyAuthenticationException.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hive.service.auth.ldap;
+
+import org.apache.hive.service.auth.HttpAuthenticationException;
+
+public class HttpEmptyAuthenticationException extends HttpAuthenticationException {
+
+ public HttpEmptyAuthenticationException(String msg) {
+ super(msg);
+ }
+}
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
index 70ffa3c6a3a..a89fbc16183 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
@@ -18,6 +18,7 @@
package org.apache.hive.service.cli.thrift;
+import java.io.EOFException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
@@ -36,6 +37,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.NewCookie;
+import com.google.common.io.ByteStreams;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -53,6 +55,7 @@ import org.apache.hive.service.auth.HiveAuthFactory;
import org.apache.hive.service.auth.HttpAuthUtils;
import org.apache.hive.service.auth.HttpAuthenticationException;
import org.apache.hive.service.auth.PasswdAuthenticationProvider;
+import org.apache.hive.service.auth.ldap.HttpEmptyAuthenticationException;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.session.SessionManager;
import org.apache.thrift.TProcessor;
@@ -207,7 +210,19 @@ public class ThriftHttpServlet extends TServlet {
super.doPost(request, response);
}
catch (HttpAuthenticationException e) {
- LOG.error("Error: ", e);
+ // Ignore HttpEmptyAuthenticationException, it is normal for knox
+ // to send a request with empty header
+ if (!(e instanceof HttpEmptyAuthenticationException)) {
+ LOG.error("Error: ", e);
+ }
+ // Wait until all the data is received and then respond with 401
+ if (request.getContentLength() < 0) {
+ try {
+ ByteStreams.skipFully(request.getInputStream(), Integer.MAX_VALUE);
+ } catch (EOFException ex) {
+ LOG.info(ex.getMessage());
+ }
+ }
// Send a 401 to the client
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
if(isKerberosAuthMode(authType)) {
@@ -404,6 +419,9 @@ public class ThriftHttpServlet extends TServlet {
try {
return serviceUGI.doAs(new HttpKerberosServerAction(request, serviceUGI));
} catch (Exception e) {
+ if (e.getCause() instanceof HttpEmptyAuthenticationException) {
+ throw (HttpEmptyAuthenticationException)e.getCause();
+ }
LOG.error("Failed to authenticate with hive/_HOST kerberos principal");
throw new HttpAuthenticationException(e);
}
@@ -546,7 +564,7 @@ public class ThriftHttpServlet extends TServlet {
String authHeader = request.getHeader(HttpAuthUtils.AUTHORIZATION);
// Each http request must have an Authorization header
if (authHeader == null || authHeader.isEmpty()) {
- throw new HttpAuthenticationException("Authorization header received " +
+ throw new HttpEmptyAuthenticationException("Authorization header received " +
"from the client is empty.");
}