You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2015/11/24 22:26:56 UTC
ambari git commit: AMBARI-14050 - Produce a Warning When Jetty Pool
Size Is Too Low And Increase It (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk 7f1e2f670 -> 5c6f8a402
AMBARI-14050 - Produce a Warning When Jetty Pool Size Is Too Low And Increase It (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5c6f8a40
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5c6f8a40
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5c6f8a40
Branch: refs/heads/trunk
Commit: 5c6f8a4029476fb466ad2513200db487b744e189
Parents: 7f1e2f6
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Tue Nov 24 15:36:12 2015 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Nov 24 16:26:44 2015 -0500
----------------------------------------------------------------------
.../ambari/server/controller/AmbariServer.java | 203 ++++++++++++-------
.../server/controller/AmbariServerTest.java | 32 ++-
2 files changed, 161 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c6f8a40/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 56034d9..bd7ac48 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -19,16 +19,19 @@
package org.apache.ambari.server.controller;
-import com.google.common.util.concurrent.ServiceManager;
-import com.google.gson.Gson;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Scopes;
-import com.google.inject.Singleton;
-import com.google.inject.name.Named;
-import com.google.inject.persist.Transactional;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
+import java.io.File;
+import java.io.IOException;
+import java.net.Authenticator;
+import java.net.BindException;
+import java.net.PasswordAuthentication;
+import java.net.URL;
+import java.util.EnumSet;
+import java.util.Enumeration;
+import java.util.Map;
+
+import javax.crypto.BadPaddingException;
+import javax.servlet.DispatcherType;
+
import org.apache.ambari.eventdb.webservice.WorkflowJsonService;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.StateRecoveryManager;
@@ -81,7 +84,6 @@ import org.apache.ambari.server.orm.entities.MetainfoEntity;
import org.apache.ambari.server.resources.ResourceManager;
import org.apache.ambari.server.resources.api.rest.GetResource;
import org.apache.ambari.server.scheduler.ExecutionScheduleManager;
-import org.apache.ambari.server.security.AmbariEntryPoint;
import org.apache.ambari.server.security.AmbariServerSecurityHeaderFilter;
import org.apache.ambari.server.security.CertificateManager;
import org.apache.ambari.server.security.SecurityFilter;
@@ -105,7 +107,6 @@ import org.apache.ambari.server.utils.RetryHelper;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.ambari.server.view.ViewRegistry;
import org.apache.velocity.app.Velocity;
-import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.SessionIdManager;
import org.eclipse.jetty.server.SessionManager;
@@ -129,17 +130,16 @@ import org.springframework.web.context.request.RequestContextListener;
import org.springframework.web.context.support.GenericWebApplicationContext;
import org.springframework.web.filter.DelegatingFilterProxy;
-import javax.crypto.BadPaddingException;
-import javax.servlet.DispatcherType;
-import java.io.File;
-import java.io.IOException;
-import java.net.Authenticator;
-import java.net.BindException;
-import java.net.PasswordAuthentication;
-import java.net.URL;
-import java.util.EnumSet;
-import java.util.Enumeration;
-import java.util.Map;
+import com.google.common.util.concurrent.ServiceManager;
+import com.google.gson.Gson;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+import com.google.inject.persist.Transactional;
+import com.sun.jersey.spi.container.servlet.ServletContainer;
@Singleton
public class AmbariServer {
@@ -191,14 +191,19 @@ public class AmbariServer {
@Inject
Configuration configs;
+
@Inject
CertificateManager certMan;
+
@Inject
Injector injector;
+
@Inject
AmbariMetaInfo ambariMetaInfo;
+
@Inject
MetainfoDAO metainfoDAO;
+
@Inject
@Named("dbInitNeeded")
boolean dbInitNeeded;
@@ -263,8 +268,8 @@ public class AmbariServer {
parentSpringAppContext.refresh();
ConfigurableListableBeanFactory factory = parentSpringAppContext.
getBeanFactory();
- factory.registerSingleton("guiceInjector",
- injector);
+
+ factory.registerSingleton("guiceInjector", injector);
factory.registerSingleton("passwordEncoder",
injector.getInstance(PasswordEncoder.class));
factory.registerSingleton("ambariLocalUserService",
@@ -280,12 +285,10 @@ public class AmbariServer {
factory.registerSingleton("ambariJwtAuthenticationFilter",
injector.getInstance(JwtAuthenticationFilter.class));
- //Spring Security xml config depends on this Bean
-
+ // Spring Security xml config depends on this Bean
String[] contextLocations = {SPRING_CONTEXT_LOCATION};
ClassPathXmlApplicationContext springAppContext = new
ClassPathXmlApplicationContext(contextLocations, parentSpringAppContext);
- //setting ambari web context
ServletContextHandler root = new ServletContextHandler(
ServletContextHandler.SECURITY | ServletContextHandler.SESSIONS);
@@ -294,6 +297,7 @@ public class AmbariServer {
configureSessionManager(sessionManager);
root.getSessionHandler().setSessionManager(sessionManager);
+ // setting ambari web context
GenericWebApplicationContext springWebAppContext = new GenericWebApplicationContext();
springWebAppContext.setServletContext(root.getServletContext());
springWebAppContext.setParent(springAppContext);
@@ -308,6 +312,7 @@ public class AmbariServer {
// and does not use sessions.
ServletContextHandler agentroot = new ServletContextHandler(
serverForAgent, "/", ServletContextHandler.NO_SESSIONS);
+
if (configs.isAgentApiGzipped()) {
configureHandlerCompression(agentroot);
}
@@ -322,15 +327,15 @@ public class AmbariServer {
// Conditionally adds security-related headers to all HTTP responses.
root.addFilter(new FilterHolder(injector.getInstance(AmbariServerSecurityHeaderFilter.class)), "/*", DISPATCHER_TYPES);
- //session-per-request strategy for api and agents
+
+ // session-per-request strategy for api
root.addFilter(new FilterHolder(injector.getInstance(AmbariPersistFilter.class)), "/api/*", DISPATCHER_TYPES);
- // root.addFilter(new FilterHolder(injector.getInstance(AmbariPersistFilter.class)), "/proxy/*", DISPATCHER_TYPES);
root.addFilter(new FilterHolder(new MethodOverrideFilter()), "/api/*", DISPATCHER_TYPES);
- // root.addFilter(new FilterHolder(new MethodOverrideFilter()), "/proxy/*", DISPATCHER_TYPES);
// register listener to capture request context
root.addEventListener(new RequestContextListener());
+ // session-per-request strategy for agents
agentroot.addFilter(new FilterHolder(injector.getInstance(AmbariPersistFilter.class)), "/agent/*", DISPATCHER_TYPES);
agentroot.addFilter(SecurityFilter.class, "/*", DISPATCHER_TYPES);
@@ -345,14 +350,16 @@ public class AmbariServer {
//Secured connector for 2-way auth
SslContextFactory contextFactoryTwoWay = new SslContextFactory();
disableInsecureProtocols(contextFactoryTwoWay);
- SslSelectChannelConnector sslConnectorTwoWay = new
- SslSelectChannelConnector(contextFactoryTwoWay);
+
+ SslSelectChannelConnector sslConnectorTwoWay = new SslSelectChannelConnector(contextFactoryTwoWay);
sslConnectorTwoWay.setPort(configs.getTwoWayAuthPort());
- String keystore = configsMap.get(Configuration.SRVR_KSTR_DIR_KEY) +
- File.separator + configsMap.get(Configuration.KSTR_NAME_KEY);
- String truststore = configsMap.get(Configuration.SRVR_KSTR_DIR_KEY) +
- File.separator + configsMap.get(Configuration.TSTR_NAME_KEY);
+ String keystore = configsMap.get(Configuration.SRVR_KSTR_DIR_KEY) + File.separator
+ + configsMap.get(Configuration.KSTR_NAME_KEY);
+
+ String truststore = configsMap.get(Configuration.SRVR_KSTR_DIR_KEY) + File.separator
+ + configsMap.get(Configuration.TSTR_NAME_KEY);
+
String srvrCrtPass = configsMap.get(Configuration.SRVR_CRT_PASS_KEY);
sslConnectorTwoWay.setKeystore(keystore);
sslConnectorTwoWay.setTruststore(truststore);
@@ -378,26 +385,42 @@ public class AmbariServer {
//Secured connector for 1-way auth
SslSelectChannelConnector sslConnectorOneWay = new SslSelectChannelConnector(contextFactoryOneWay);
sslConnectorOneWay.setPort(configs.getOneWayAuthPort());
- sslConnectorOneWay.setAcceptors(2);
- sslConnectorTwoWay.setAcceptors(2);
- serverForAgent.setConnectors(new Connector[]{sslConnectorOneWay, sslConnectorTwoWay});
+
+ // because there are two connectors sharing the same pool, cut each's
+ // acceptors in half
+ int sslAcceptors = sslConnectorOneWay.getAcceptors();
+ sslConnectorOneWay.setAcceptors(Math.max(2, sslAcceptors / 2));
+ sslConnectorTwoWay.setAcceptors(Math.max(2, sslAcceptors / 2));
+
+ // Agent Jetty thread pool
+ configureJettyThreadPool(serverForAgent, sslConnectorOneWay.getAcceptors(),
+ "qtp-ambari-agent", configs.getAgentThreadPoolSize());
+
+ serverForAgent.addConnector(sslConnectorOneWay);
+ serverForAgent.addConnector(sslConnectorTwoWay);
} else {
SelectChannelConnector agentConnector = new SelectChannelConnector();
agentConnector.setPort(configs.getOneWayAuthPort());
agentConnector.setMaxIdleTime(configs.getConnectionMaxIdleTime());
+
+ // Agent Jetty thread pool
+ configureJettyThreadPool(serverForAgent, agentConnector.getAcceptors(), "qtp-ambari-agent",
+ configs.getAgentThreadPoolSize());
+
serverForAgent.addConnector(agentConnector);
}
ServletHolder sh = new ServletHolder(ServletContainer.class);
sh.setInitParameter("com.sun.jersey.config.property.resourceConfigClass",
"com.sun.jersey.api.core.PackagesResourceConfig");
+
sh.setInitParameter("com.sun.jersey.config.property.packages",
"org.apache.ambari.server.api.rest;" +
"org.apache.ambari.server.api.services;" +
"org.apache.ambari.eventdb.webservice;" +
"org.apache.ambari.server.api");
- sh.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature",
- "true");
+
+ sh.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature", "true");
root.addServlet(sh, "/api/v1/*");
sh.setInitOrder(2);
@@ -406,7 +429,6 @@ public class AmbariServer {
viewRegistry.readViewArchives();
handlerList.addHandler(root);
-
server.setHandler(handlerList);
ServletHolder agent = new ServletHolder(ServletContainer.class);
@@ -414,8 +436,7 @@ public class AmbariServer {
"com.sun.jersey.api.core.PackagesResourceConfig");
agent.setInitParameter("com.sun.jersey.config.property.packages",
"org.apache.ambari.server.agent.rest;" + "org.apache.ambari.server.api");
- agent.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature",
- "true");
+ agent.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature", "true");
agentroot.addServlet(agent, "/agent/v1/*");
agent.setInitOrder(3);
@@ -427,22 +448,11 @@ public class AmbariServer {
"com.sun.jersey.api.core.PackagesResourceConfig");
cert.setInitParameter("com.sun.jersey.config.property.packages",
"org.apache.ambari.server.security.unsecured.rest;" + "org.apache.ambari.server.api");
- cert.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature",
- "true");
+
+ cert.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature", "true");
agentroot.addServlet(cert, "/*");
cert.setInitOrder(4);
- /*
- ServletHolder proxy = new ServletHolder(ServletContainer.class);
- proxy.setInitParameter("com.sun.jersey.config.property.resourceConfigClass",
- "com.sun.jersey.api.core.PackagesResourceConfig");
- proxy.setInitParameter("com.sun.jersey.config.property.packages",
- "org.apache.ambari.server.proxy");
- proxy.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature", "true");
- root.addServlet(proxy, "/proxy/*");
- proxy.setInitOrder(5);
- */
-
ServletHolder resources = new ServletHolder(ServletContainer.class);
resources.setInitParameter("com.sun.jersey.config.property.resourceConfigClass",
"com.sun.jersey.api.core.PackagesResourceConfig");
@@ -454,19 +464,8 @@ public class AmbariServer {
if (configs.csrfProtectionEnabled()) {
sh.setInitParameter("com.sun.jersey.spi.container.ContainerRequestFilters",
"org.apache.ambari.server.api.AmbariCsrfProtectionFilter");
- /* proxy.setInitParameter("com.sun.jersey.spi.container.ContainerRequestFilters",
- "org.apache.ambari.server.api.AmbariCsrfProtectionFilter"); */
}
- // Set jetty thread pool
- QueuedThreadPool qtp = new QueuedThreadPool(configs.getAgentThreadPoolSize());
- qtp.setName("qtp-ambari-agent");
- serverForAgent.setThreadPool(qtp);
-
- qtp = new QueuedThreadPool(configs.getClientThreadPoolSize());
- qtp.setName("qtp-client");
- server.setThreadPool(qtp);
-
/* Configure the API server to use the NIO connectors */
SelectChannelConnector apiConnector;
@@ -498,6 +497,8 @@ public class AmbariServer {
apiConnector.setMaxIdleTime(configs.getConnectionMaxIdleTime());
}
+ // Client Jetty thread pool
+ configureJettyThreadPool(server, apiConnector.getAcceptors(), "qtp-ambari-client", configs.getClientThreadPoolSize());
server.addConnector(apiConnector);
server.setStopAtShutdown(true);
@@ -515,6 +516,7 @@ public class AmbariServer {
Clusters clusters = injector.getInstance(Clusters.class);
StringBuilder clusterDump = new StringBuilder();
clusters.debugDump(clusterDump);
+
LOG.info("********* Current Clusters State *********");
LOG.info(clusterDump.toString());
@@ -523,6 +525,7 @@ public class AmbariServer {
LOG.info("********* Initializing ActionManager **********");
ActionManager manager = injector.getInstance(ActionManager.class);
+
LOG.info("********* Initializing Controller **********");
AmbariManagementController controller = injector.getInstance(
AmbariManagementController.class);
@@ -531,11 +534,11 @@ public class AmbariServer {
ExecutionScheduleManager executionScheduleManager = injector
.getInstance(ExecutionScheduleManager.class);
-
clusterController = controller;
StateRecoveryManager recoveryManager = injector.getInstance(
StateRecoveryManager.class);
+
recoveryManager.doWork();
/*
@@ -569,6 +572,64 @@ public class AmbariServer {
}
/**
+ * The Jetty thread pool consists of three basic types of threads:
+ * <ul>
+ * <li>Acceptors</li>
+ * <li>Selectors</li>
+ * <li>Threads which can actually do stuff</li>
+ * <ul>
+ * The {@link SelectChannelConnector} uses the
+ * {@link Runtime#availableProcessors()} as a way to determine how many
+ * acceptors and selectors to create. If the number of processors is too
+ * great, then there will be no threads left to fullfil connection requests.
+ * This method ensures that the pool size is configured correctly, taking into
+ * account the number of available processors (sockets x core x
+ * threads-per-core).
+ * <p/>
+ * If the configured pool size is determined to be too small, then this will
+ * log a warning and increase the pool size to ensure that there are at least
+ * 20 available threads for requests.
+ *
+ * @param server
+ * the Jetty server instance which will have the threadpool set on it
+ * (not {@code null}).
+ * @param acceptorThreads
+ * the number of Acceptor threads configured for the connector.
+ * @param threadPoolName
+ * the name of the thread pool being configured (not {@code null}).
+ * @param configuredThreadPoolSize
+ * the size of the pool from {@link Configuration}.
+ */
+ protected void configureJettyThreadPool(Server server, int acceptorThreads,
+ String threadPoolName, int configuredThreadPoolSize) {
+ int minumumAvailableThreads = 20;
+
+ // multiply by two since there is 1 selector for every acceptor
+ int reservedJettyThreads = acceptorThreads * 2;
+
+ // this is the calculation used by Jetty
+ if (configuredThreadPoolSize < reservedJettyThreads + minumumAvailableThreads) {
+ int newThreadPoolSize = reservedJettyThreads + minumumAvailableThreads;
+
+ LOG.warn(
+ "The configured Jetty {} thread pool value of {} is not sufficient on a host with {} processors. Increasing the value to {}.",
+ threadPoolName, configuredThreadPoolSize, Runtime.getRuntime().availableProcessors(),
+ newThreadPoolSize);
+
+ configuredThreadPoolSize = newThreadPoolSize;
+ }
+
+ LOG.info(
+ "Jetty is configuring {} with {} reserved acceptors/selectors and a total pool size of {} for {} processors.",
+ threadPoolName, acceptorThreads * 2, configuredThreadPoolSize,
+ Runtime.getRuntime().availableProcessors());
+
+ QueuedThreadPool qtp = new QueuedThreadPool(configuredThreadPoolSize);
+ qtp.setName(threadPoolName);
+ server.setThreadPool(qtp);
+ }
+
+ /**
* Disables insecure protocols and cipher suites (exact list is defined
* at server properties)
*/
@@ -727,7 +788,7 @@ public class AmbariServer {
* Initialize the view registry singleton instance.
*/
public void initViewRegistry() {
- ViewRegistry.initInstance(this.viewRegistry);
+ ViewRegistry.initInstance(viewRegistry);
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c6f8a40/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
index 621010a..02941b5 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
@@ -29,16 +29,21 @@ import java.net.InetAddress;
import java.net.PasswordAuthentication;
import java.util.EnumSet;
+import javax.servlet.DispatcherType;
+import javax.servlet.SessionCookieConfig;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.orm.GuiceJpaInitializer;
import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
import org.apache.velocity.app.Velocity;
import org.easymock.EasyMock;
+import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.SessionManager;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlets.GzipFilter;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -47,9 +52,6 @@ import org.junit.Test;
import com.google.inject.Guice;
import com.google.inject.Injector;
-import javax.servlet.DispatcherType;
-import javax.servlet.SessionCookieConfig;
-
public class AmbariServerTest {
private Injector injector;
@@ -156,4 +158,28 @@ public class AmbariServerTest {
EasyMock.verify(handler);
}
+ /**
+ * Tests that Jetty pools are configured with the correct number of
+ * Acceptor/Selector threads.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testJettyThreadPoolCalculation() throws Exception {
+ Server server = new Server();
+ AmbariServer ambariServer = new AmbariServer();
+
+ // 12 acceptors (48 core machine) with a configured pool size of 25
+ ambariServer.configureJettyThreadPool(server, 12, "mock-pool", 25);
+ Assert.assertEquals(44, ((QueuedThreadPool) server.getThreadPool()).getMaxThreads());
+
+ // 2 acceptors (8 core machine) with a configured pool size of 25
+ ambariServer.configureJettyThreadPool(server, 2, "mock-pool", 25);
+ Assert.assertEquals(25, ((QueuedThreadPool) server.getThreadPool()).getMaxThreads());
+
+ // 16 acceptors (64 core machine) with a configured pool size of 35
+ ambariServer.configureJettyThreadPool(server, 16, "mock-pool", 35);
+ Assert.assertEquals(52, ((QueuedThreadPool) server.getThreadPool()).getMaxThreads());
+
+ }
}