You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ai...@apache.org on 2017/03/30 18:16:37 UTC
[1/2] hive git commit: HIVE-16049: upgrade to jetty 9 (Sean Busbey & Aihua Xu, reviewed by Sergio Peña)
Repository: hive
Updated Branches:
refs/heads/master bbc874eeb -> e339d1444
HIVE-16049: upgrade to jetty 9 (Sean Busbey & Aihua Xu, reviewed by Sergio Pe�a)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/191302cf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/191302cf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/191302cf
Branch: refs/heads/master
Commit: 191302cf4f9eae5ef51964bdab8d8e859292aa17
Parents: bbc874e
Author: Aihua Xu <ai...@apache.org>
Authored: Thu Mar 16 09:23:47 2017 -0400
Committer: Aihua Xu <ai...@apache.org>
Committed: Thu Mar 30 14:01:28 2017 -0400
----------------------------------------------------------------------
common/pom.xml | 44 ++++++---
.../java/org/apache/hive/http/HttpServer.java | 69 ++++++++------
hcatalog/pom.xml | 22 +++++
hcatalog/webhcat/svr/pom.xml | 74 +++++++++++++--
.../apache/hive/hcatalog/templeton/Main.java | 37 +++-----
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 4 +-
pom.xml | 28 +++++-
service/pom.xml | 98 ++++++++++++++++++--
.../cli/thrift/ThriftHttpCLIService.java | 33 ++++---
.../hive/service/server/TestHS2HttpServer.java | 2 +-
10 files changed, 315 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 8474a87..dd5542e 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -69,20 +69,24 @@
<version>${jline.version}</version>
</dependency>
<dependency>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- <version>${jetty.version}</version>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-rewrite</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
</dependency>
<dependency>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- <version>${javax-servlet.version}</version>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
@@ -129,6 +133,18 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
@@ -149,6 +165,10 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/common/src/java/org/apache/hive/http/HttpServer.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java
index db5650d..fd3d457 100644
--- a/common/src/java/org/apache/hive/http/HttpServer.java
+++ b/common/src/java/org/apache/hive/http/HttpServer.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.util.Shell;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.Appender;
@@ -54,11 +53,13 @@ import org.apache.logging.log4j.core.appender.OutputStreamManager;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.rewrite.handler.RewriteRegexRule;
import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.LowResourceMonitor;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.handler.ContextHandler.Context;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
-import org.eclipse.jetty.server.nio.SelectChannelConnector;
-import org.eclipse.jetty.server.ssl.SslSelectChannelConnector;
+import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.DefaultServlet;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.FilterMapping;
@@ -85,9 +86,9 @@ public class HttpServer {
public static final String ADMINS_ACL = "admins.acl";
private final String name;
- private final String appDir;
- private final WebAppContext webAppContext;
- private final Server webServer;
+ private String appDir;
+ private WebAppContext webAppContext;
+ private Server webServer;
/**
* Create a status server on the given port.
@@ -95,16 +96,7 @@ public class HttpServer {
private HttpServer(final Builder b) throws IOException {
this.name = b.name;
- webServer = new Server();
- appDir = getWebAppsPath(b.name);
- webAppContext = createWebAppContext(b);
-
- if (b.useSPNEGO) {
- // Secure the web server with kerberos
- setupSpnegoFilter(b);
- }
-
- initializeWebServer(b);
+ createWebServer(b);
}
public static class Builder {
@@ -219,7 +211,7 @@ public class HttpServer {
}
public int getPort() {
- return webServer.getConnectors()[0].getLocalPort();
+ return ((ServerConnector)(webServer.getConnectors()[0])).getLocalPort();
}
/**
@@ -345,9 +337,14 @@ public class HttpServer {
* Create a channel connector for "http/https" requests
*/
Connector createChannelConnector(int queueSize, Builder b) {
- SelectChannelConnector connector;
+ ServerConnector connector;
+
+ final HttpConfiguration conf = new HttpConfiguration();
+ conf.setRequestHeaderSize(1024*64);
+ final HttpConnectionFactory http = new HttpConnectionFactory(conf);
+
if (!b.useSSL) {
- connector = new SelectChannelConnector();
+ connector = new ServerConnector(webServer, http);
} else {
SslContextFactory sslContextFactory = new SslContextFactory();
sslContextFactory.setKeyStorePath(b.keyStorePath);
@@ -357,15 +354,13 @@ public class HttpServer {
sslContextFactory.addExcludeProtocols(excludedSSLProtocols.toArray(
new String[excludedSSLProtocols.size()]));
sslContextFactory.setKeyStorePassword(b.keyStorePassword);
- connector = new SslSelectChannelConnector(sslContextFactory);
+ connector = new ServerConnector(webServer, sslContextFactory, http);
}
- connector.setLowResourcesMaxIdleTime(10000);
connector.setAcceptQueueSize(queueSize);
- connector.setResolveNames(false);
- connector.setUseDirectBuffers(false);
- connector.setRequestHeaderSize(1024*64);
connector.setReuseAddress(true);
+ connector.setHost(b.host);
+ connector.setPort(b.port);
return connector;
}
@@ -378,7 +373,7 @@ public class HttpServer {
}
}
- void initializeWebServer(Builder b) {
+ private void createWebServer(final Builder b) throws IOException {
// Create the thread pool for the web server to handle HTTP requests
QueuedThreadPool threadPool = new QueuedThreadPool();
if (b.maxThreads > 0) {
@@ -386,12 +381,26 @@ public class HttpServer {
}
threadPool.setDaemon(true);
threadPool.setName(b.name + "-web");
- webServer.setThreadPool(threadPool);
- // Create the channel connector for the web server
- Connector connector = createChannelConnector(threadPool.getMaxThreads(), b);
- connector.setHost(b.host);
- connector.setPort(b.port);
+ this.webServer = new Server(threadPool);
+ this.appDir = getWebAppsPath(b.name);
+ this.webAppContext = createWebAppContext(b);
+
+ if (b.useSPNEGO) {
+ // Secure the web server with kerberos
+ setupSpnegoFilter(b);
+ }
+
+ initializeWebServer(b, threadPool.getMaxThreads());
+ }
+
+ private void initializeWebServer(final Builder b, int queueSize) {
+ // Set handling for low resource conditions.
+ final LowResourceMonitor low = new LowResourceMonitor(webServer);
+ low.setLowResourcesIdleTimeout(10000);
+ webServer.addBean(low);
+
+ Connector connector = createChannelConnector(queueSize, b);
webServer.addConnector(connector);
RewriteHandler rwHandler = new RewriteHandler();
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/hcatalog/pom.xml b/hcatalog/pom.xml
index 34de177..9bb82c1 100644
--- a/hcatalog/pom.xml
+++ b/hcatalog/pom.xml
@@ -71,6 +71,28 @@
<version>${pig.version}</version>
<classifier>h2</classifier>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/hcatalog/webhcat/svr/pom.xml
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml
index c5ad387..032f017 100644
--- a/hcatalog/webhcat/svr/pom.xml
+++ b/hcatalog/webhcat/svr/pom.xml
@@ -45,9 +45,50 @@
<artifactId>hive-hcatalog-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-runner</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-sslengine</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- inter-project -->
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-rewrite</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<version>${jersey.version}</version>
@@ -93,11 +134,6 @@
<version>${jackson.version}</version>
</dependency>
<dependency>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all-server</artifactId>
- <version>${jetty.version}</version>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>jul-to-slf4j</artifactId>
<version>${slf4j.version}</version>
@@ -107,7 +143,7 @@
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.version}</version>
<exclusions>
- <exclusion>
+ <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
@@ -121,16 +157,42 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- test inter-project -->
<dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
index 5208bf4..3ed3ece 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
@@ -25,6 +25,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.HashMap;
import org.slf4j.Logger;
@@ -43,14 +44,15 @@ import org.eclipse.jetty.rewrite.handler.RedirectPatternRule;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.servlet.FilterHolder;
-import org.eclipse.jetty.servlet.FilterMapping;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.xml.XmlConfiguration;
import org.slf4j.bridge.SLF4JBridgeHandler;
+import javax.servlet.DispatcherType;
import javax.servlet.http.HttpServletRequest;
/**
@@ -122,7 +124,7 @@ public class Main {
checkEnv();
runServer(port);
// Currently only print the first port to be consistent with old behavior
- port = ArrayUtils.isEmpty(server.getConnectors()) ? -1 : server.getConnectors()[0].getPort();
+ port = ArrayUtils.isEmpty(server.getConnectors()) ? -1 : ((ServerConnector)(server.getConnectors()[0])).getLocalPort();
System.out.println("templeton: listening on port " + port);
LOG.info("Templeton listening on port " + port);
@@ -185,6 +187,7 @@ public class Main {
// Add the Auth filter
FilterHolder fHolder = makeAuthFilter();
+ EnumSet<DispatcherType> dispatches = EnumSet.of(DispatcherType.REQUEST);
/*
* We add filters for each of the URIs supported by templeton.
@@ -193,28 +196,18 @@ public class Main {
* This is because mapreduce does not use secure credentials for
* callbacks. So jetty would fail the request as unauthorized.
*/
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/ddl/*",
- FilterMapping.REQUEST);
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/pig/*",
- FilterMapping.REQUEST);
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*",
- FilterMapping.REQUEST);
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/sqoop/*",
- FilterMapping.REQUEST);
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*",
- FilterMapping.REQUEST);
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*",
- FilterMapping.REQUEST);
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*",
- FilterMapping.REQUEST);
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*",
- FilterMapping.REQUEST);
- root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/version/*",
- FilterMapping.REQUEST);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/ddl/*", dispatches);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/pig/*", dispatches);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*", dispatches);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/sqoop/*", dispatches);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", dispatches);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*", dispatches);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/mapreduce/*", dispatches);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/status/*", dispatches);
+ root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/version/*", dispatches);
if (conf.getBoolean(AppConfig.XSRF_FILTER_ENABLED, false)){
- root.addFilter(makeXSRFFilter(), "/" + SERVLET_PATH + "/*",
- FilterMapping.REQUEST);
+ root.addFilter(makeXSRFFilter(), "/" + SERVLET_PATH + "/*", dispatches);
LOG.debug("XSRF filter enabled");
} else {
LOG.warn("XSRF filter disabled");
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 22e5ee8..4a4fef0 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -76,7 +76,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.eclipse.jetty.server.ssl.SslSocketConnector;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.joda.time.DateTime;
import org.json.JSONException;
import org.json.JSONObject;
@@ -378,7 +378,7 @@ public class LlapServiceDriver {
LlapTezUtils.class, // llap-tez
LlapInputFormat.class, // llap-server
HiveInputFormat.class, // hive-exec
- SslSocketConnector.class, // hive-common (https deps)
+ SslContextFactory.class, // hive-common (https deps)
RegistryUtils.ServiceRecordMarshal.class, // ZK registry
// log4j2
com.lmax.disruptor.RingBuffer.class, // disruptor
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6a1a501..6b46129 100644
--- a/pom.xml
+++ b/pom.xml
@@ -159,10 +159,10 @@
<jamon.plugin.version>2.3.4</jamon.plugin.version>
<jamon-runtime.version>2.3.1</jamon-runtime.version>
<javaewah.version>0.3.2</javaewah.version>
- <javax-servlet.version>3.0.0.v201112011016</javax-servlet.version>
+ <javax-servlet.version>3.1.0</javax-servlet.version>
<javolution.version>5.5.1</javolution.version>
<jdo-api.version>3.0.1</jdo-api.version>
- <jetty.version>7.6.0.v20120127</jetty.version>
+ <jetty.version>9.3.8.v20160314</jetty.version>
<jersey.version>1.14</jersey.version>
<!-- Glassfish jersey is included for Spark client test only -->
<glassfish.jersey.version>2.22.2</glassfish.jersey.version>
@@ -610,11 +610,31 @@
<version>${jackson.version}</version>
</dependency>
<dependency>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all-server</artifactId>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-rewrite</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>${javax-servlet.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.datanucleus</groupId>
<artifactId>datanucleus-api-jdo</artifactId>
<version>${datanucleus-api-jdo.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/service/pom.xml
----------------------------------------------------------------------
diff --git a/service/pom.xml b/service/pom.xml
index 9306739..7c0fc4c 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -53,6 +53,16 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-llap-server</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- inter-project -->
@@ -70,8 +80,8 @@
<groupId>net.sf.jpam</groupId>
<artifactId>jpam</artifactId>
<version>${jpam.version}</version>
- <exclusions>
- <exclusion>
+ <exclusions>
+ <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
@@ -81,6 +91,25 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-runner</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
<!-- used by thrift generated code -->
<dependency>
<groupId>commons-lang</groupId>
@@ -88,19 +117,26 @@
<version>${commons-lang.version}</version>
</dependency>
<dependency>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- <version>${jetty.version}</version>
- </dependency>
- <dependency>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
<version>${jasper.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
<version>${jasper.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
@@ -127,12 +163,60 @@
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.jamon</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index ebec165..c4d4e02 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -38,8 +38,10 @@ import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TServlet;
-import org.eclipse.jetty.server.nio.SelectChannelConnector;
-import org.eclipse.jetty.server.ssl.SslSelectChannelConnector;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.FilterMapping;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
@@ -63,9 +65,6 @@ public class ThriftHttpCLIService extends ThriftCLIService {
@Override
public void run() {
try {
- // HTTP Server
- httpServer = new org.eclipse.jetty.server.Server();
-
// Server thread pool
// Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
String threadPoolName = "HiveServer2-HttpHandler-Pool";
@@ -73,19 +72,26 @@ public class ThriftHttpCLIService extends ThriftCLIService {
maxWorkerThreads, workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new ThreadFactoryWithGarbageCleanup(threadPoolName), oomHook);
ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService);
- httpServer.setThreadPool(threadPool);
- // Connector configs
- SelectChannelConnector connector = new SelectChannelConnector();
+ // HTTP Server
+ httpServer = new Server(threadPool);
+
+
+ ServerConnector connector;
+
+ final HttpConfiguration conf = new HttpConfiguration();
// Configure header size
int requestHeaderSize =
hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE);
int responseHeaderSize =
hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE);
- connector.setRequestHeaderSize(requestHeaderSize);
- connector.setResponseHeaderSize(responseHeaderSize);
+ conf.setRequestHeaderSize(requestHeaderSize);
+ conf.setResponseHeaderSize(responseHeaderSize);
+ final HttpConnectionFactory http = new HttpConnectionFactory(conf);
+
boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
String schemeName = useSsl ? "https" : "http";
+
// Change connector if SSL is used
if (useSsl) {
String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim();
@@ -103,14 +109,17 @@ public class ThriftHttpCLIService extends ThriftCLIService {
Arrays.toString(sslContextFactory.getExcludeProtocols()));
sslContextFactory.setKeyStorePath(keyStorePath);
sslContextFactory.setKeyStorePassword(keyStorePassword);
- connector = new SslSelectChannelConnector(sslContextFactory);
+ connector = new ServerConnector(httpServer, sslContextFactory, http);
+ } else {
+ connector = new ServerConnector(httpServer, http);
}
+
connector.setPort(portNum);
// Linux:yes, Windows:no
connector.setReuseAddress(true);
int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME,
TimeUnit.MILLISECONDS);
- connector.setMaxIdleTime(maxIdleTime);
+ connector.setIdleTimeout(maxIdleTime);
httpServer.addConnector(connector);
http://git-wip-us.apache.org/repos/asf/hive/blob/191302cf/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java
index 4d50dd9..663a266 100644
--- a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java
+++ b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java
@@ -142,7 +142,7 @@ public class TestHS2HttpServer {
private String getURLResponseAsString(String baseURL) throws IOException {
URL url = new URL(baseURL);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertEquals("Got an HTTP response code other thank OK.", HttpURLConnection.HTTP_OK, conn.getResponseCode());
StringWriter writer = new StringWriter();
IOUtils.copy(conn.getInputStream(), writer, "UTF-8");
return writer.toString();
[2/2] hive git commit: HIVE-16057: SchemaTool ignores --passWord
argument if hadoop.security.credential.provider.path is configured (Peter
Vary, reviewed by Aihua Xu)
Posted by ai...@apache.org.
HIVE-16057: SchemaTool ignores --passWord argument if hadoop.security.credential.provider.path is configured (Peter Vary, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e339d144
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e339d144
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e339d144
Branch: refs/heads/master
Commit: e339d144425f01bad2aaab1367bded15f5dbaaba
Parents: 191302c
Author: Aihua Xu <ai...@apache.org>
Authored: Thu Mar 30 14:08:08 2017 -0400
Committer: Aihua Xu <ai...@apache.org>
Committed: Thu Mar 30 14:08:08 2017 -0400
----------------------------------------------------------------------
.../org/apache/hive/beeline/HiveSchemaTool.java | 17 ++++++++++-------
.../org/apache/hive/beeline/TestSchemaTool.java | 5 +++++
2 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e339d144/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
index 4255ebd..7dd4d5f 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
@@ -90,13 +90,6 @@ public class HiveSchemaTool {
this.hiveConf = hiveConf;
this.dbType = dbType;
this.metaStoreSchemaInfo = new MetaStoreSchemaInfo(hiveHome, dbType);
- userName = hiveConf.get(ConfVars.METASTORE_CONNECTION_USER_NAME.varname);
- try {
- passWord = ShimLoader.getHadoopShims().getPassword(hiveConf,
- HiveConf.ConfVars.METASTOREPWD.varname);
- } catch (IOException err) {
- throw new HiveMetaException("Error getting metastore password", err);
- }
}
public HiveConf getHiveConf() {
@@ -1118,9 +1111,19 @@ public class HiveSchemaTool {
if (line.hasOption("userName")) {
schemaTool.setUserName(line.getOptionValue("userName"));
+ } else {
+ schemaTool.setUserName(
+ schemaTool.getHiveConf().get(ConfVars.METASTORE_CONNECTION_USER_NAME.varname));
}
if (line.hasOption("passWord")) {
schemaTool.setPassWord(line.getOptionValue("passWord"));
+ } else {
+ try {
+ schemaTool.setPassWord(ShimLoader.getHadoopShims().getPassword(schemaTool.getHiveConf(),
+ HiveConf.ConfVars.METASTOREPWD.varname));
+ } catch (IOException err) {
+ throw new HiveMetaException("Error getting metastore password", err);
+ }
}
if (line.hasOption("dryRun")) {
schemaTool.setDryRun(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/e339d144/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
index 22630b9..57bbe0e 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
@@ -36,6 +36,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser;
@@ -57,6 +58,10 @@ public class TestSchemaTool extends TestCase {
hiveConf = new HiveConf(this.getClass());
schemaTool = new HiveSchemaTool(
System.getProperty("test.tmp.dir", "target/tmp"), hiveConf, "derby");
+ schemaTool.setUserName(
+ schemaTool.getHiveConf().get(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME.varname));
+ schemaTool.setPassWord(ShimLoader.getHadoopShims().getPassword(schemaTool.getHiveConf(),
+ HiveConf.ConfVars.METASTOREPWD.varname));
System.setProperty("beeLine.system.exit", "true");
errStream = System.err;
outStream = System.out;