You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/06/04 20:43:08 UTC
ambari git commit: AMBARI-11678 Perf: Add GZIP support for Ambari
Server API (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk a4f9081c7 -> 6a0d1e0a2
AMBARI-11678 Perf: Add GZIP support for Ambari Server API (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6a0d1e0a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6a0d1e0a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6a0d1e0a
Branch: refs/heads/trunk
Commit: 6a0d1e0a2dbf241fd729d4d50f4f09e45e17baf6
Parents: a4f9081
Author: Dmytro Sen <ds...@apache.org>
Authored: Thu Jun 4 21:41:38 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Thu Jun 4 21:41:38 2015 +0300
----------------------------------------------------------------------
.../src/main/python/ambari_agent/Controller.py | 3 +-
.../src/main/python/ambari_agent/security.py | 72 ++++++++++++--------
ambari-project/pom.xml | 5 ++
ambari-server/pom.xml | 4 ++
.../server/configuration/Configuration.java | 39 ++++++++++-
.../ambari/server/controller/AmbariServer.java | 29 +++++++-
.../server/controller/AmbariServerTest.java | 30 +++++++-
7 files changed, 148 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a0d1e0a/ambari-agent/src/main/python/ambari_agent/Controller.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Controller.py b/ambari-agent/src/main/python/ambari_agent/Controller.py
index 94b574a..4e5de6c 100644
--- a/ambari-agent/src/main/python/ambari_agent/Controller.py
+++ b/ambari-agent/src/main/python/ambari_agent/Controller.py
@@ -393,7 +393,8 @@ class Controller(threading.Thread):
try:
if self.cachedconnect is None: # Lazy initialization
self.cachedconnect = security.CachedHTTPSConnection(self.config)
- req = urllib2.Request(url, data, {'Content-Type': 'application/json'})
+ req = urllib2.Request(url, data, {'Content-Type': 'application/json',
+ 'Accept-encoding': 'gzip'})
response = self.cachedconnect.request(req)
return json.loads(response)
except Exception, exception:
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a0d1e0a/ambari-agent/src/main/python/ambari_agent/security.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/security.py b/ambari-agent/src/main/python/ambari_agent/security.py
index a86d06b..bfaf134 100644
--- a/ambari-agent/src/main/python/ambari_agent/security.py
+++ b/ambari-agent/src/main/python/ambari_agent/security.py
@@ -15,7 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from StringIO import StringIO
+import gzip
import httplib
import urllib2
import socket
@@ -31,8 +32,9 @@ import platform
logger = logging.getLogger(__name__)
-GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s'+os.sep+'%(hostname)s.key" '\
- '-subj /OU=%(hostname)s/ -out "%(keysdir)s'+os.sep+'%(hostname)s.csr"'
+GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s' \
+ + os.sep + '%(hostname)s.key" -subj /OU=%(hostname)s/ ' \
+ '-out "%(keysdir)s' + os.sep + '%(hostname)s.csr"'
class VerifiedHTTPSConnection(httplib.HTTPSConnection):
@@ -44,9 +46,11 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection):
def connect(self):
self.two_way_ssl_required = self.config.isTwoWaySSLConnection()
- logger.debug("Server two-way SSL authentication required: %s" % str(self.two_way_ssl_required))
+ logger.debug("Server two-way SSL authentication required: %s" % str(
+ self.two_way_ssl_required))
if self.two_way_ssl_required is True:
- logger.info('Server require two-way SSL authentication. Use it instead of one-way...')
+ logger.info(
+ 'Server require two-way SSL authentication. Use it instead of one-way...')
if not self.two_way_ssl_required:
try:
@@ -56,8 +60,9 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection):
'turned off on the server.')
except (ssl.SSLError, AttributeError):
self.two_way_ssl_required = True
- logger.info('Insecure connection to https://' + self.host + ':' + self.port +
- '/ failed. Reconnecting using two-way SSL authentication..')
+ logger.info(
+ 'Insecure connection to https://' + self.host + ':' + self.port +
+ '/ failed. Reconnecting using two-way SSL authentication..')
if self.two_way_ssl_required:
self.certMan = CertificateManager(self.config)
@@ -70,21 +75,21 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection):
try:
self.sock = ssl.wrap_socket(sock,
- keyfile=agent_key,
- certfile=agent_crt,
- cert_reqs=ssl.CERT_REQUIRED,
- ca_certs=server_crt)
+ keyfile=agent_key,
+ certfile=agent_crt,
+ cert_reqs=ssl.CERT_REQUIRED,
+ ca_certs=server_crt)
logger.info('SSL connection established. Two-way SSL authentication '
'completed successfully.')
except ssl.SSLError as err:
logger.error('Two-way SSL authentication failed. Ensure that '
- 'server and agent certificates were signed by the same CA '
- 'and restart the agent. '
- '\nIn order to receive a new agent certificate, remove '
- 'existing certificate file from keys directory. As a '
- 'workaround you can turn off two-way SSL authentication in '
- 'server configuration(ambari.properties) '
- '\nExiting..')
+ 'server and agent certificates were signed by the same CA '
+ 'and restart the agent. '
+ '\nIn order to receive a new agent certificate, remove '
+ 'existing certificate file from keys directory. As a '
+ 'workaround you can turn off two-way SSL authentication in '
+ 'server configuration(ambari.properties) '
+ '\nExiting..')
raise err
def create_connection(self):
@@ -112,13 +117,15 @@ class CachedHTTPSConnection:
def connect(self):
if not self.connected:
- self.httpsconn = VerifiedHTTPSConnection(self.server, self.port, self.config)
+ self.httpsconn = VerifiedHTTPSConnection(self.server, self.port,
+ self.config)
self.httpsconn.connect()
self.connected = True
# possible exceptions are caught and processed in Controller
def forceClear(self):
- self.httpsconn = VerifiedHTTPSConnection(self.server, self.port, self.config)
+ self.httpsconn = VerifiedHTTPSConnection(self.server, self.port,
+ self.config)
self.connect()
def request(self, req):
@@ -127,6 +134,10 @@ class CachedHTTPSConnection:
self.httpsconn.request(req.get_method(), req.get_full_url(),
req.get_data(), req.headers)
response = self.httpsconn.getresponse()
+ # Ungzip if gzipped
+ if response.getheader('Content-Encoding') == 'gzip':
+ buf = StringIO(response.read())
+ response = gzip.GzipFile(fileobj=buf)
readResponse = response.read()
except Exception as ex:
# This exception is caught later in Controller
@@ -144,7 +155,7 @@ class CertificateManager():
self.keysdir = os.path.abspath(self.config.get('security', 'keysdir'))
self.server_crt = self.config.get('security', 'server_crt')
self.server_url = 'https://' + hostname.server_hostname(config) + ':' \
- + self.config.get('server', 'url_port')
+ + self.config.get('server', 'url_port')
def getAgentKeyName(self):
keysdir = os.path.abspath(self.config.get('security', 'keysdir'))
@@ -164,7 +175,8 @@ class CertificateManager():
def checkCertExists(self):
- s = os.path.abspath(self.config.get('security', 'keysdir')) + os.sep + "ca.crt"
+ s = os.path.abspath(
+ self.config.get('security', 'keysdir')) + os.sep + "ca.crt"
server_crt_exists = os.path.exists(s)
@@ -202,18 +214,20 @@ class CertificateManager():
srvr_crt_f.write(response)
def reqSignCrt(self):
- sign_crt_req_url = self.server_url + '/certs/' + hostname.hostname(self.config)
+ sign_crt_req_url = self.server_url + '/certs/' + hostname.hostname(
+ self.config)
agent_crt_req_f = open(self.getAgentCrtReqName())
agent_crt_req_content = agent_crt_req_f.read()
passphrase_env_var = self.config.get('security', 'passphrase_env_var_name')
passphrase = os.environ[passphrase_env_var]
register_data = {'csr': agent_crt_req_content,
- 'passphrase': passphrase}
+ 'passphrase': passphrase}
data = json.dumps(register_data)
proxy_handler = urllib2.ProxyHandler({})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
- req = urllib2.Request(sign_crt_req_url, data, {'Content-Type': 'application/json'})
+ req = urllib2.Request(sign_crt_req_url, data,
+ {'Content-Type': 'application/json'})
f = urllib2.urlopen(req)
response = f.read()
f.close()
@@ -239,14 +253,16 @@ class CertificateManager():
raise ssl.SSLError
def genAgentCrtReq(self):
- generate_script = GEN_AGENT_KEY % {'hostname': hostname.hostname(self.config),
- 'keysdir' : os.path.abspath(self.config.get('security', 'keysdir'))}
+ generate_script = GEN_AGENT_KEY % {
+ 'hostname': hostname.hostname(self.config),
+ 'keysdir': os.path.abspath(self.config.get('security', 'keysdir'))}
logger.info(generate_script)
if platform.system() == 'Windows':
p = subprocess.Popen(generate_script, stdout=subprocess.PIPE)
p.communicate()
else:
- p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE)
+ p = subprocess.Popen([generate_script], shell=True,
+ stdout=subprocess.PIPE)
p.communicate()
def initSecurity(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a0d1e0a/ambari-project/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml
index 528dbf4..99e950a 100644
--- a/ambari-project/pom.xml
+++ b/ambari-project/pom.xml
@@ -248,6 +248,11 @@
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlets</artifactId>
+ <version>8.1.17.v20150415</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
<version>8.1.17.v20150415</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a0d1e0a/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index a17e529..6aa62f2 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -1594,6 +1594,10 @@
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlets</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
</dependency>
<!--jsp support for jetty -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a0d1e0a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 4536b3c..e8457d7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -84,6 +84,9 @@ public class Configuration {
public static final String API_AUTHENTICATE = "api.authenticate";
public static final String API_USE_SSL = "api.ssl";
public static final String API_CSRF_PREVENTION_KEY = "api.csrfPrevention.enabled";
+ public static final String API_GZIP_COMPRESSION_ENABLED_KEY = "api.gzip.compression.enabled";
+ public static final String API_GZIP_MIN_COMPRESSION_SIZE_KEY = "api.gzip.compression.min.size";
+ public static final String AGENT_API_GZIP_COMPRESSION_ENABLED_KEY = "agent.api.gzip.compression.enabled";
public static final String SRVR_TWO_WAY_SSL_KEY = "security.server.two_way_ssl";
public static final String SRVR_TWO_WAY_SSL_PORT_KEY = "security.server.two_way_ssl.port";
public static final String SRVR_ONE_WAY_SSL_PORT_KEY = "security.server.one_way_ssl.port";
@@ -293,6 +296,8 @@ public class Configuration {
private static final String SRVR_TWO_WAY_SSL_DEFAULT = "false";
private static final String SRVR_KSTR_DIR_DEFAULT = ".";
private static final String API_CSRF_PREVENTION_DEFAULT = "true";
+ private static final String API_GZIP_COMPRESSION_ENABLED_DEFAULT = "true";
+ private static final String API_GZIP_MIN_COMPRESSION_SIZE_DEFAULT = "10240";
private static final String SRVR_CRT_PASS_FILE_DEFAULT = "pass.txt";
private static final String SRVR_CRT_PASS_LEN_DEFAULT = "50";
private static final String SRVR_DISABLED_CIPHERS_DEFAULT = "";
@@ -892,7 +897,8 @@ public class Configuration {
* @return int
*/
public int getClientSSLApiPort() {
- return Integer.parseInt(properties.getProperty(CLIENT_API_SSL_PORT_KEY, String.valueOf(CLIENT_API_SSL_PORT_DEFAULT)));
+ return Integer.parseInt(properties.getProperty(CLIENT_API_SSL_PORT_KEY,
+ String.valueOf(CLIENT_API_SSL_PORT_DEFAULT)));
}
/**
@@ -915,6 +921,37 @@ public class Configuration {
}
/**
+ * Check to see if the API responses should be compressed via gzip or not
+ * @return false if not, true if gzip compression needs to be used.
+ */
+ public boolean isApiGzipped() {
+ return "true".equalsIgnoreCase(properties.getProperty(
+ API_GZIP_COMPRESSION_ENABLED_KEY,
+ API_GZIP_COMPRESSION_ENABLED_DEFAULT));
+ }
+
+ /**
+ * Check to see if the agent API responses should be compressed via gzip or not
+ * @return false if not, true if gzip compression needs to be used.
+ */
+ public boolean isAgentApiGzipped() {
+ return "true".equalsIgnoreCase(properties.getProperty(
+ AGENT_API_GZIP_COMPRESSION_ENABLED_KEY,
+ API_GZIP_COMPRESSION_ENABLED_DEFAULT));
+ }
+
+ /**
+ * Check to see if the API responses should be compressed via gzip or not
+ * Content will only be compressed if content length is either unknown or
+ * greater this value
+ * @return false if not, true if ssl needs to be used.
+ */
+ public String getApiGzipMinSize() {
+ return properties.getProperty(API_GZIP_MIN_COMPRESSION_SIZE_KEY,
+ API_GZIP_MIN_COMPRESSION_SIZE_DEFAULT);
+ }
+
+ /**
* Check persistence type Ambari Server should use. Possible values:
* in-memory - use in-memory Derby database to store data
* local - use local Postgres instance
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a0d1e0a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 8320715..e430c98 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -107,6 +107,7 @@ import org.eclipse.jetty.server.ssl.SslSelectChannelConnector;
import org.eclipse.jetty.servlet.DefaultServlet;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlets.GzipFilter;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.util.thread.QueuedThreadPool;
@@ -274,6 +275,9 @@ public class AmbariServer {
// and does not use sessions.
ServletContextHandler agentroot = new ServletContextHandler(
serverForAgent, "/", ServletContextHandler.NO_SESSIONS);
+ if (configs.isAgentApiGzipped()) {
+ configureHandlerCompression(agentroot);
+ }
ServletHolder rootServlet = root.addServlet(DefaultServlet.class, "/");
rootServlet.setInitParameter("dirAllowed", "false");
@@ -529,12 +533,13 @@ public class AmbariServer {
}
/**
- * Performs basic configuration of root handler with static values and values from
- * configuration file.
+ * Performs basic configuration of root handler with static values and values
+ * from configuration file.
*
* @param root root handler
*/
protected void configureRootHandler(ServletContextHandler root) {
+ configureHandlerCompression(root);
root.setContextPath(CONTEXT_PATH);
root.setErrorHandler(injector.getInstance(AmbariErrorHandler.class));
root.setMaxFormContentSize(-1);
@@ -544,6 +549,26 @@ public class AmbariServer {
}
/**
+ * Performs GZIP compression configuration of the context handler
+ * with static values and values from configuration file
+ *
+ * @param context handler
+ */
+ protected void configureHandlerCompression(ServletContextHandler context) {
+ if (configs.isApiGzipped()) {
+ FilterHolder gzipFilter = context.addFilter(GzipFilter.class, "/*",
+ EnumSet.of(DispatcherType.REQUEST));
+
+ gzipFilter.setInitParameter("methods","GET,POST,PUT,DELETE");
+ gzipFilter.setInitParameter("mimeTypes",
+ "text/html,text/plain,text/xml,text/css,application/x-javascript," +
+ "application/xml,application/x-www-form-urlencoded," +
+ "application/javascript,application/json");
+ gzipFilter.setInitParameter("minGzipSize", configs.getApiGzipMinSize());
+ }
+ }
+
+ /**
* Performs basic configuration of session manager with static values and values from
* configuration file.
*
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a0d1e0a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
index 6e1c97e..621010a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.controller;
+import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
@@ -26,6 +27,7 @@ import static org.easymock.EasyMock.verify;
import java.net.Authenticator;
import java.net.InetAddress;
import java.net.PasswordAuthentication;
+import java.util.EnumSet;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.configuration.Configuration;
@@ -34,7 +36,9 @@ import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
import org.apache.velocity.app.Velocity;
import org.easymock.EasyMock;
import org.eclipse.jetty.server.SessionManager;
+import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlets.GzipFilter;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -43,6 +47,7 @@ import org.junit.Test;
import com.google.inject.Guice;
import com.google.inject.Injector;
+import javax.servlet.DispatcherType;
import javax.servlet.SessionCookieConfig;
public class AmbariServerTest {
@@ -119,15 +124,36 @@ public class AmbariServerTest {
@Test
public void testConfigureRootHandler() throws Exception {
- final ServletContextHandler handler = EasyMock.createNiceMock(ServletContextHandler.class);
+ final ServletContextHandler handler =
+ EasyMock.createNiceMock(ServletContextHandler.class);
+ final FilterHolder filter = EasyMock.createNiceMock(FilterHolder.class);
handler.setMaxFormContentSize(-1);
EasyMock.expectLastCall().once();
- replay(handler);
+ EasyMock.expect(handler.addFilter(GzipFilter.class, "/*",
+ EnumSet.of(DispatcherType.REQUEST))).andReturn(filter).once();
+ replay(handler, filter);
injector.getInstance(AmbariServer.class).configureRootHandler(handler);
EasyMock.verify(handler);
}
+ @Test
+ public void testConfigureCompression() throws Exception {
+ final ServletContextHandler handler =
+ EasyMock.createNiceMock(ServletContextHandler.class);
+ final FilterHolder filter = EasyMock.createNiceMock(FilterHolder.class);
+
+ EasyMock.expect(handler.addFilter(GzipFilter.class, "/*",
+ EnumSet.of(DispatcherType.REQUEST))).andReturn(filter).once();
+ filter.setInitParameter(anyObject(String.class),anyObject(String.class));
+ EasyMock.expectLastCall().times(3);
+ replay(handler, filter);
+
+ injector.getInstance(AmbariServer.class).configureHandlerCompression(handler);
+
+ EasyMock.verify(handler);
+ }
+
}