You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by tm...@apache.org on 2018/10/05 21:39:20 UTC
[8/8] impala git commit: IMPALA-7660: Support ECDH ciphers for debug
webserver
IMPALA-7660: Support ECDH ciphers for debug webserver
A recent change (IMPALA-7519) added support for ecdh ciphers for the
beeswax/hs2 server. This patch pulls in a recent change on squeasel to
extend that support to the debug webserver.
It also fixes a bug that prevented start-impala-cluster.py from
completing successfully when the webserver is launched with ssl, due
to it trying to verify the availablitiy of the webserver over http.
Testing:
- Added a custom cluster test that verifies start-impala-cluster.py
runs successfully with webserver ssl enabled.
- Adds the webserver to an existing test for ecdh ciphers.
Change-Id: I80a6b370d5860812cde13229b5bcb2977814c73c
Reviewed-on: http://gerrit.cloudera.org:8080/11585
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/0e1de31b
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/0e1de31b
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/0e1de31b
Branch: refs/heads/master
Commit: 0e1de31ba56bdac73b8db5c5ff316334c84725d9
Parents: 37bee3a
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Mon Oct 1 22:04:05 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Fri Oct 5 21:16:17 2018 +0000
----------------------------------------------------------------------
be/src/thirdparty/squeasel/squeasel.c | 36 ++++++++++++++++++++++++----
tests/common/impala_cluster.py | 15 ++++++++----
tests/common/impala_service.py | 26 +++++++++++++-------
tests/custom_cluster/test_client_ssl.py | 34 +++++++++++++++++++++-----
4 files changed, 86 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/0e1de31b/be/src/thirdparty/squeasel/squeasel.c
----------------------------------------------------------------------
diff --git a/be/src/thirdparty/squeasel/squeasel.c b/be/src/thirdparty/squeasel/squeasel.c
index 2149497..045740d 100644
--- a/be/src/thirdparty/squeasel/squeasel.c
+++ b/be/src/thirdparty/squeasel/squeasel.c
@@ -4298,11 +4298,37 @@ static int set_ssl_option(struct sq_context *ctx) {
(void) SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, pem);
}
- if (ctx->config[SSL_CIPHERS] != NULL &&
- (SSL_CTX_set_cipher_list(ctx->ssl_ctx, ctx->config[SSL_CIPHERS]) == 0)) {
- cry(fc(ctx), "SSL_CTX_set_cipher_list: error setting ciphers (%s): %s",
- ctx->config[SSL_CIPHERS], ssl_error());
- return 0;
+ if (ctx->config[SSL_CIPHERS] != NULL) {
+ if (SSL_CTX_set_cipher_list(ctx->ssl_ctx, ctx->config[SSL_CIPHERS]) == 0) {
+ cry(fc(ctx), "SSL_CTX_set_cipher_list: error setting ciphers (%s): %s",
+ ctx->config[SSL_CIPHERS], ssl_error());
+ return 0;
+ }
+#ifndef OPENSSL_NO_ECDH
+#if OPENSSL_VERSION_NUMBER < 0x10002000L
+ // OpenSSL 1.0.1 and below only support setting a single ECDH curve at once.
+ // We choose prime256v1 because it's the first curve listed in the "modern
+ // compatibility" section of the Mozilla Server Side TLS recommendations,
+ // accessed Feb. 2017.
+ EC_KEY* ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
+ if (ecdh == NULL) {
+ cry(fc(ctx), "EC_KEY_new_by_curve_name: %s", ssl_error());
+ }
+
+ int rc = SSL_CTX_set_tmp_ecdh(ctx->ssl_ctx, ecdh);
+ if (rc <= 0) {
+ cry(fc(ctx), "SSL_CTX_set_tmp_ecdh: %s", ssl_error());
+ }
+#elif OPENSSL_VERSION_NUMBER < 0x10100000L
+ // OpenSSL 1.0.2 provides the set_ecdh_auto API which internally figures out
+ // the best curve to use.
+ int rc = SSL_CTX_set_ecdh_auto(ctx->ssl_ctx, 1);
+ if (rc <= 0) {
+ cry(fc(ctx), "SSL_CTX_set_ecdh_auto: %s", ssl_error());
+ }
+#endif
+#endif
+
}
return 1;
http://git-wip-us.apache.org/repos/asf/impala/blob/0e1de31b/tests/common/impala_cluster.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index f25c8ed..05ff172 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -222,6 +222,10 @@ class BaseImpalaProcess(Process):
def _get_webserver_port(self, default=None):
return int(self._get_arg_value('webserver_port', default))
+ def _get_webserver_ssl(self):
+ """Returns true if the webserver is being run with ssl."""
+ return self._get_arg_value("webserver_certificate_file", "") != ""
+
def _get_arg_value(self, arg_name, default=None):
"""Gets the argument value for given argument name"""
for arg in self.cmd:
@@ -239,7 +243,8 @@ class ImpaladProcess(BaseImpalaProcess):
self.service = ImpaladService(self.hostname, self._get_webserver_port(default=25000),
self.__get_beeswax_port(default=21000),
self.__get_be_port(default=22000),
- self.__get_hs2_port(default=21050))
+ self.__get_hs2_port(default=21050),
+ self._get_webserver_ssl())
def __get_beeswax_port(self, default=None):
return int(self._get_arg_value('beeswax_port', default))
@@ -264,16 +269,16 @@ class ImpaladProcess(BaseImpalaProcess):
class StateStoreProcess(BaseImpalaProcess):
def __init__(self, cmd):
super(StateStoreProcess, self).__init__(cmd, socket.gethostname())
- self.service =\
- StateStoredService(self.hostname, self._get_webserver_port(default=25010))
+ self.service = StateStoredService(
+ self.hostname, self._get_webserver_port(default=25010), self._get_webserver_ssl())
# Represents a catalogd process
class CatalogdProcess(BaseImpalaProcess):
def __init__(self, cmd):
super(CatalogdProcess, self).__init__(cmd, socket.gethostname())
- self.service = CatalogdService(self.hostname,
- self._get_webserver_port(default=25020), self.__get_port(default=26000))
+ self.service = CatalogdService(self.hostname, self._get_webserver_port(default=25020),
+ self._get_webserver_ssl(), self.__get_port(default=26000))
def __get_port(self, default=None):
return int(self._get_arg_value('catalog_service_port', default))
http://git-wip-us.apache.org/repos/asf/impala/blob/0e1de31b/tests/common/impala_service.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index 0ad4496..a99e0ce 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -22,6 +22,7 @@
import json
import logging
import re
+import ssl
import urllib
from time import sleep, time
@@ -41,17 +42,24 @@ LOG.setLevel(level=logging.DEBUG)
# Base class for all Impala services
# TODO: Refactor the retry/timeout logic into a common place.
class BaseImpalaService(object):
- def __init__(self, hostname, webserver_port):
+ def __init__(self, hostname, webserver_port, webserver_ssl=False):
self.hostname = hostname
self.webserver_port = webserver_port
+ self.webserver_ssl = webserver_ssl
def open_debug_webpage(self, page_name, timeout=10, interval=1):
start_time = time()
while (time() - start_time < timeout):
try:
- return urllib.urlopen("http://%s:%d/%s" %
- (self.hostname, int(self.webserver_port), page_name))
+ protocol = "http"
+ context = None
+ if self.webserver_ssl:
+ protocol = "https"
+ context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ url = "%s://%s:%d/%s" % \
+ (protocol, self.hostname, int(self.webserver_port), page_name)
+ return urllib.urlopen(url, context=context)
except Exception:
LOG.info("Debug webpage not yet available.")
sleep(interval)
@@ -166,8 +174,8 @@ class BaseImpalaService(object):
# new connections or accessing the debug webpage.
class ImpaladService(BaseImpalaService):
def __init__(self, hostname, webserver_port=25000, beeswax_port=21000, be_port=22000,
- hs2_port=21050):
- super(ImpaladService, self).__init__(hostname, webserver_port)
+ hs2_port=21050, webserver_ssl=False):
+ super(ImpaladService, self).__init__(hostname, webserver_port, webserver_ssl)
self.beeswax_port = beeswax_port
self.be_port = be_port
self.hs2_port = hs2_port
@@ -307,8 +315,8 @@ class ImpaladService(BaseImpalaService):
# Allows for interacting with the StateStore service to perform operations such as
# accessing the debug webpage.
class StateStoredService(BaseImpalaService):
- def __init__(self, hostname, webserver_port):
- super(StateStoredService, self).__init__(hostname, webserver_port)
+ def __init__(self, hostname, webserver_port, webserver_ssl):
+ super(StateStoredService, self).__init__(hostname, webserver_port, webserver_ssl)
def wait_for_live_subscribers(self, num_subscribers, timeout=15, interval=1):
self.wait_for_metric_value('statestore.live-backends', num_subscribers,
@@ -318,8 +326,8 @@ class StateStoredService(BaseImpalaService):
# Allows for interacting with the Catalog service to perform operations such as
# accessing the debug webpage.
class CatalogdService(BaseImpalaService):
- def __init__(self, hostname, webserver_port, service_port):
- super(CatalogdService, self).__init__(hostname, webserver_port)
+ def __init__(self, hostname, webserver_port, webserver_ssl, service_port):
+ super(CatalogdService, self).__init__(hostname, webserver_port, webserver_ssl)
self.service_port = service_port
def get_catalog_version(self, timeout=10, interval=1):
http://git-wip-us.apache.org/repos/asf/impala/blob/0e1de31b/tests/custom_cluster/test_client_ssl.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index b6f7e04..7bbc2df 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -19,6 +19,7 @@
import logging
import os
import pytest
+import requests
import signal
import ssl
import socket
@@ -110,13 +111,27 @@ class TestClientSsl(CustomClusterTestSuite):
assert "Query Status: Cancelled" in result.stdout
assert impalad.wait_for_num_in_flight_queries(0)
+ WEBSERVER_SSL_ARGS = ("--webserver_certificate_file=%(cert_dir)s/server-cert.pem "
+ "--webserver_private_key_file=%(cert_dir)s/server-key.pem "
+ % {'cert_dir': CERT_DIR})
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(impalad_args=WEBSERVER_SSL_ARGS,
+ statestored_args=WEBSERVER_SSL_ARGS,
+ catalogd_args=WEBSERVER_SSL_ARGS)
+ def test_webserver_ssl(self):
+ "Tests that the debug web pages are reachable when run with ssl."
+ self._verify_ssl_webserver()
+
# Test that the shell can connect to a ECDH only cluster.
- TLS_ECDH_ARGS = ("--ssl_client_ca_certificate=%s/server-cert.pem "
- "--ssl_server_certificate=%s/server-cert.pem "
- "--ssl_private_key=%s/server-key.pem "
- "--hostname=localhost " # Required to match hostname in certificate"
- "--ssl_cipher_list=ECDHE-RSA-AES128-GCM-SHA256 "
- % (CERT_DIR, CERT_DIR, CERT_DIR))
+ TLS_ECDH_ARGS = ("--ssl_client_ca_certificate=%(cert_dir)s/server-cert.pem "
+ "--ssl_server_certificate=%(cert_dir)s/server-cert.pem "
+ "--ssl_private_key=%(cert_dir)s/server-key.pem "
+ "--hostname=localhost " # Required to match hostname in certificate"
+ "--ssl_cipher_list=ECDHE-RSA-AES128-GCM-SHA256 "
+ "--webserver_certificate_file=%(cert_dir)s/server-cert.pem "
+ "--webserver_private_key_file=%(cert_dir)s/server-key.pem "
+ % {'cert_dir': CERT_DIR})
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(impalad_args=TLS_ECDH_ARGS,
@@ -128,6 +143,7 @@ class TestClientSsl(CustomClusterTestSuite):
def test_tls_ecdh(self, vector):
self._verify_negative_cases()
self._validate_positive_cases("%s/server-cert.pem" % self.CERT_DIR)
+ self._verify_ssl_webserver()
# Test that the shell can connect to a TLS1.2 only cluster, and for good measure
# restrict the cipher suite to just one choice.
@@ -209,3 +225,9 @@ class TestClientSsl(CustomClusterTestSuite):
result = run_impala_shell_cmd(shell_options)
for msg in [self.SSL_ENABLED, self.CONNECTED, self.FETCHED]:
assert msg in result.stderr
+
+ def _verify_ssl_webserver(self):
+ for port in ["25000", "25010", "25020"]:
+ url = "https://localhost:%s" % port
+ response = requests.get(url, verify="%s/server-cert.pem" % self.CERT_DIR)
+ assert response.status_code == requests.codes.ok, url