You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by he...@apache.org on 2017/08/16 16:18:41 UTC
[1/3] incubator-impala git commit: IMPALA-5775: Allow shell to
support TLSv1, v1.1 and v1.2
Repository: incubator-impala
Updated Branches:
refs/heads/master 8149dbfc9 -> 294d42adc
IMPALA-5775: Allow shell to support TLSv1, v1.1 and v1.2
The shell uses Thrift's TSSLSocket to negotiate secure connections to
Impala. This socket uses a variable SSL_VERSION to determine which SSL
and TLS protocol versions it will connect to.
SSL_VERSION was hardcoded to be PROTOCOL_TLSv1, which only supports
TLSv1 servers and no other protocol version. Change the allowed version
to be PROTOCOL_SSLv23, which supports any TLS or SSL protocol. We rely
on the server not to allow SSLv2 or v3 connections.
Testing: Added a new custom cluster test to confirm that the shell can
connect to a TLSv1.2 cluster. Confirmed that the test is correctly
skipped on machines with an old version of OpenSSL that does not support
TLSv1.2.
Change-Id: I5487f82d110676b9c3c7a5305931da00c7f68ca0
Reviewed-on: http://gerrit.cloudera.org:8080/7675
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/e4a0e2f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/e4a0e2f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/e4a0e2f3
Branch: refs/heads/master
Commit: e4a0e2f391bce3b8411ce7e5010856a54dc52991
Parents: 8149dbf
Author: Henry Robinson <he...@cloudera.com>
Authored: Mon Aug 14 18:04:55 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Aug 16 08:10:02 2017 +0000
----------------------------------------------------------------------
shell/TSSLSocketWithWildcardSAN.py | 7 ++++++
tests/custom_cluster/test_client_ssl.py | 33 ++++++++++++++++++++++++++--
tests/util/thrift_util.py | 3 +++
3 files changed, 41 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e4a0e2f3/shell/TSSLSocketWithWildcardSAN.py
----------------------------------------------------------------------
diff --git a/shell/TSSLSocketWithWildcardSAN.py b/shell/TSSLSocketWithWildcardSAN.py
old mode 100644
new mode 100755
index 7e15ac8..d65cc68
--- a/shell/TSSLSocketWithWildcardSAN.py
+++ b/shell/TSSLSocketWithWildcardSAN.py
@@ -18,6 +18,7 @@
# under the License.
import re
+import ssl
from thrift.transport import TSSLSocket
from thrift.transport.TTransport import TTransportException
@@ -42,6 +43,12 @@ class TSSLSocketWithWildcardSAN(TSSLSocket.TSSLSocket):
ca_certs=None,
unix_socket=None):
TSSLSocket.TSSLSocket.__init__(self, host, port, validate, ca_certs, unix_socket)
+ # Set client protocol choice to be very permissive, as we rely on servers to enforce
+ # good protocol selection. This value is forwarded to the ssl.wrap_socket() API during
+ # open(). See https://docs.python.org/2/library/ssl.html#socket-creation for a table
+ # that shows a better option is not readily available for sockets that use
+ # wrap_socket().
+ self.SSL_VERSION = ssl.PROTOCOL_SSLv23
def _validate_cert(self):
cert = self.handle.getpeercert()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e4a0e2f3/tests/custom_cluster/test_client_ssl.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index 9d19234..4858ce0 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -20,6 +20,7 @@ import logging
import os
import pytest
import signal
+import ssl
import socket
import time
@@ -28,6 +29,17 @@ from tests.common.impala_service import ImpaladService
from tests.shell.util import run_impala_shell_cmd, run_impala_shell_cmd_no_expect, \
ImpalaShell
+REQUIRED_MIN_OPENSSL_VERSION = 0x10001000L
+HAS_LEGACY_OPENSSL = True
+SKIP_SSL_MSG = "Legacy OpenSSL module detected"
+try:
+ HAS_LEGACY_OPENSSL = ssl.OPENSSL_VERSION_NUMBER < REQUIRED_MIN_OPENSSL_VERSION
+ SKIP_SSL_MSG = "Only have OpenSSL version %X, but test requires %X" % (
+ ssl.OPENSSL_VERSION_NUMBER, REQUIRED_MIN_OPENSSL_VERSION)
+except AttributeError:
+ # Old ssl module versions don't even have OPENSSL_VERSION_NUMBER as a member.
+ pass
+
class TestClientSsl(CustomClusterTestSuite):
"""Tests for a client using SSL (particularly, the Impala Shell) """
@@ -92,11 +104,28 @@ class TestClientSsl(CustomClusterTestSuite):
assert "Query Status: Cancelled" in result.stdout
assert impalad.wait_for_num_in_flight_queries(0)
+ # Test that the shell can connect to a TLS1.2 only cluster, and for good measure
+ # restrict the cipher suite to just one choice.
+ TLS_V12_ARGS = ("--ssl_server_certificate=%s/wildcard-cert.pem "
+ "--ssl_private_key=%s/wildcard-cert.key "
+ "--ssl_minimum_version=tlsv1.2 "
+ "--ssl_client_ca_certificate=%s/wildcardCA.pem "
+ "--ssl_cipher_list=AES128-GCM-SHA256 "
+ % (CERT_DIR, CERT_DIR, CERT_DIR))
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(impalad_args=TLS_V12_ARGS,
+ statestored_args=TLS_V12_ARGS,
+ catalogd_args=TLS_V12_ARGS)
+ @pytest.mark.skipif(HAS_LEGACY_OPENSSL, reason=SKIP_SSL_MSG)
+ def test_tls_v12(self, vector):
+ self._validate_positive_cases("%s/wildcardCA.pem" % self.CERT_DIR)
+
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(impalad_args=SSL_WILDCARD_ARGS,
statestored_args=SSL_WILDCARD_ARGS,
catalogd_args=SSL_WILDCARD_ARGS)
- @pytest.mark.xfail(run=True, reason="IMPALA-4295 on Centos6")
+ @pytest.mark.skipif(HAS_LEGACY_OPENSSL, reason=SKIP_SSL_MSG)
def test_wildcard_ssl(self, vector):
""" Test for IMPALA-3159: Test with a certificate which has a wildcard for the
CommonName.
@@ -109,7 +138,7 @@ class TestClientSsl(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args(impalad_args=SSL_WILDCARD_SAN_ARGS,
statestored_args=SSL_WILDCARD_SAN_ARGS,
catalogd_args=SSL_WILDCARD_SAN_ARGS)
- @pytest.mark.xfail(run=True, reason="IMPALA-4295")
+ @pytest.mark.skipif(HAS_LEGACY_OPENSSL, reason=SKIP_SSL_MSG)
def test_wildcard_san_ssl(self, vector):
""" Test for IMPALA-3159: Test with a certificate which has a wildcard as a SAN. """
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e4a0e2f3/tests/util/thrift_util.py
----------------------------------------------------------------------
diff --git a/tests/util/thrift_util.py b/tests/util/thrift_util.py
index 6f5871b..24e5fd7 100644
--- a/tests/util/thrift_util.py
+++ b/tests/util/thrift_util.py
@@ -42,6 +42,9 @@ def create_transport(host, port, service, transport_type="buffered", user=None,
sock = TSSLSocket.TSSLSocket(host, port, validate=False)
else:
sock = TSSLSocket.TSSLSocket(host, port, validate=True, ca_certs=ssl_cert)
+ # Set allowed SSL / TLS protocols to a permissive set to connect to any Impala server.
+ import ssl
+ sock.SSL_VERSION = ssl.PROTOCOL_SSLv23
else:
sock = TSocket(host, port)
if transport_type.lower() == "buffered":
[2/3] incubator-impala git commit: IMPALA-3200: More end-to-end tests
Posted by he...@apache.org.
IMPALA-3200: More end-to-end tests
* Test that query startup fails if there is insufficient reservation on
one backend.
* Verify that free buffers and clean pages shrink to zero when idle.
Change-Id: I3a3b6adf5b97b16594c6c4855d5fa831c5a90e10
Reviewed-on: http://gerrit.cloudera.org:8080/7668
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f74f665b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f74f665b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f74f665b
Branch: refs/heads/master
Commit: f74f665b6cf4425ad890c9e1e79aced883ccbf7a
Parents: e4a0e2f
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Mon Aug 14 13:57:22 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Aug 16 08:15:30 2017 +0000
----------------------------------------------------------------------
tests/custom_cluster/test_mem_reservations.py | 55 ++++++++++++++++++----
tests/verifiers/metric_verifier.py | 6 ++-
2 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f74f665b/tests/custom_cluster/test_mem_reservations.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_mem_reservations.py b/tests/custom_cluster/test_mem_reservations.py
index d8a19e8..bf1641b 100644
--- a/tests/custom_cluster/test_mem_reservations.py
+++ b/tests/custom_cluster/test_mem_reservations.py
@@ -19,6 +19,8 @@ import pytest
import threading
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.impala_test_suite import LOG
+from tests.verifiers.metric_verifier import MetricVerifier
class TestMemReservations(CustomClusterTestSuite):
"""Tests for memory reservations that require custom cluster arguments."""
@@ -28,7 +30,8 @@ class TestMemReservations(CustomClusterTestSuite):
return 'functional-query'
@pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args(impalad_args="--buffer_pool_limit=2g")
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--buffer_pool_limit=2g --memory_maintenance_sleep_time_ms=100")
def test_per_backend_min_reservation(self, vector):
"""Tests that the per-backend minimum reservations are used (IMPALA-4833).
The test sets the buffer_pool_limit very low (2gb), and then runs a query against
@@ -37,12 +40,15 @@ class TestMemReservations(CustomClusterTestSuite):
per-backend minimum reservations are not used, then one of the queries fails to
acquire its minimum reservation. This was verified to fail before IMPALA-4833, and
succeeds after.
+
+ Memory maintenance sleep time is set low so we can verify that buffers are
+ released.
"""
assert len(self.cluster.impalads) == 3
# This query will have scan fragments on all nodes, but the coordinator fragment
# has 6 analytic nodes, 5 sort nodes, and an aggregation.
- QUERY = """
+ COORDINATOR_QUERY = """
select max(t.c1), avg(t.c2), min(t.c3), avg(c4), avg(c5), avg(c6)
from (select
max(tinyint_col) over (order by int_col) c1,
@@ -54,15 +60,22 @@ class TestMemReservations(CustomClusterTestSuite):
from functional.alltypes) t;
"""
- # Set the DEFAULT_SPILLABLE_BUFFER_SIZE and MIN_SPILLABLE_BUFFER_SIZE to 64MiB
- # so that the coordinator node requires ~1.2gb and the other backends require ~200mb.
+ # This query has two grouping aggregations on each node.
+ SYMMETRIC_QUERY = """
+ select count(*)
+ from (select distinct * from functional.alltypes) v"""
+
+ # Set the DEFAULT_SPILLABLE_BUFFER_SIZE and MIN_SPILLABLE_BUFFER_SIZE to 64MiB.
+ # so that for COORDINATOR_QUERY, the coordinator node requires ~1.2gb and the
+ # other backends require ~200mb and for SYMMETRIC_QUERY all backends require
+ # ~1.05gb.
CONFIG_MAP = {'DEFAULT_SPILLABLE_BUFFER_SIZE': '67108864',
'MIN_SPILLABLE_BUFFER_SIZE': '67108864'}
- # Create two threads to submit QUERY to two different coordinators concurrently.
class QuerySubmitThread(threading.Thread):
- def __init__(self, coordinator):
+ def __init__(self, query, coordinator):
super(QuerySubmitThread, self).__init__()
+ self.query = query
self.coordinator = coordinator
self.error = None
@@ -71,7 +84,7 @@ class TestMemReservations(CustomClusterTestSuite):
try:
client.set_configuration(CONFIG_MAP)
for i in xrange(20):
- result = client.execute(QUERY)
+ result = client.execute(self.query)
assert result.success
assert len(result.data) == 1
except Exception, e:
@@ -79,8 +92,34 @@ class TestMemReservations(CustomClusterTestSuite):
finally:
client.close()
- threads = [QuerySubmitThread(self.cluster.impalads[i]) for i in xrange(2)]
+ # Create two threads to submit COORDINATOR_QUERY to two different coordinators concurrently.
+ # They should both succeed.
+ threads = [QuerySubmitThread(COORDINATOR_QUERY, self.cluster.impalads[i])
+ for i in xrange(2)]
for t in threads: t.start()
for t in threads:
t.join()
assert t.error is None
+
+ # Create two threads to submit COORDINATOR_QUERY to one coordinator and
+ # SYMMETRIC_QUERY to another coordinator. One of the queries should fail because
+ # memory would be overcommitted on daemon 0.
+ threads = [QuerySubmitThread(COORDINATOR_QUERY, self.cluster.impalads[0]),
+ QuerySubmitThread(SYMMETRIC_QUERY, self.cluster.impalads[1])]
+ for t in threads: t.start()
+ num_errors = 0
+ for t in threads:
+ t.join()
+ if t.error is not None:
+ assert "Failed to get minimum memory reservation" in t.error
+ LOG.info("Query failed with error: %s", t.error)
+ LOG.info(t.query)
+ num_errors += 1
+ assert num_errors == 1
+
+ # Check that free buffers are released over time. We set the memory maintenance sleep
+ # time very low above so this should happen quickly.
+ verifiers = [MetricVerifier(i.service) for i in self.cluster.impalads]
+ for v in verifiers:
+ v.wait_for_metric("buffer-pool.free-buffers", 0, timeout=60)
+ v.wait_for_metric("buffer-pool.free-buffer-bytes", 0, timeout=60)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f74f665b/tests/verifiers/metric_verifier.py
----------------------------------------------------------------------
diff --git a/tests/verifiers/metric_verifier.py b/tests/verifiers/metric_verifier.py
index d5753fd..3331135 100644
--- a/tests/verifiers/metric_verifier.py
+++ b/tests/verifiers/metric_verifier.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
#
-# Verfier for common impalad metrics
+# Verifier for common impalad metrics
# List of metrics that should be equal to zero when there are no outstanding queries.
METRIC_LIST = [
@@ -30,6 +30,10 @@ METRIC_LIST = [
"impala-server.num-files-open-for-insert",
# Disable checking of num-missing-volume-id due to IMPALA-467
# "impala-server.scan-ranges.num-missing-volume-id",
+ # Buffer pool pages belong to specific queries. Therefore there should be
+ # no clean pages if there are no queries running.
+ "buffer-pool.clean-pages",
+ "buffer-pool.clean-page-bytes"
]
class MetricVerifier(object):
[3/3] incubator-impala git commit: IMPALA-5769: Add periodic minidump
cleanup
Posted by he...@apache.org.
IMPALA-5769: Add periodic minidump cleanup
Minidumps can be written by sending SIGUSR1 to our daemon processes.
That way, an arbitrary number of minidump files can be created. This
change adds minidump cleanup to the periodic log file cleanup to
effectively bound the maximum number of minidumps we keep around.
Change-Id: Ie02ff2271412d814f84a4ff42ccbca51d91bf980
Reviewed-on: http://gerrit.cloudera.org:8080/7605
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/294d42ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/294d42ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/294d42ad
Branch: refs/heads/master
Commit: 294d42adc117046f975665834af03ddaa53ec27e
Parents: f74f665
Author: Lars Volker <lv...@cloudera.com>
Authored: Mon Aug 7 11:53:27 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Aug 16 08:41:16 2017 +0000
----------------------------------------------------------------------
be/src/common/init.cc | 13 +++++++----
be/src/util/minidump.cc | 9 +-------
be/src/util/minidump.h | 4 ++++
tests/custom_cluster/test_breakpad.py | 36 +++++++++++++++++++++++++-----
4 files changed, 45 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/294d42ad/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index 523796a..d778523 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -55,12 +55,13 @@
using namespace impala;
+DECLARE_bool(enable_process_lifetime_heap_profiling);
+DECLARE_string(heap_profile_dir);
DECLARE_string(hostname);
-DECLARE_string(redaction_rules_file);
-// TODO: renamed this to be more generic when we have a good CM release to do so.
+// TODO: rename this to be more generic when we have a good CM release to do so.
DECLARE_int32(logbufsecs);
-DECLARE_string(heap_profile_dir);
-DECLARE_bool(enable_process_lifetime_heap_profiling);
+DECLARE_int32(max_minidumps);
+DECLARE_string(redaction_rules_file);
DEFINE_int32(max_log_files, 10, "Maximum number of log files to retain per severity "
"level. The most recent log files are retained. If set to 0, all log files are "
@@ -121,6 +122,10 @@ static scoped_ptr<impala::Thread> pause_monitor;
impala::CheckAndRotateLogFiles(FLAGS_max_log_files);
// Check for audit event log rotation in every interval of the maintenance thread
impala::CheckAndRotateAuditEventLogFiles(FLAGS_max_audit_event_log_files);
+ // Check for minidump rotation in every interval of the maintenance thread. This is
+ // necessary since an arbitrary number of minidumps can be written by sending SIGUSR1
+ // to the process.
+ impala::CheckAndRotateMinidumps(FLAGS_max_minidumps);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/294d42ad/be/src/util/minidump.cc
----------------------------------------------------------------------
diff --git a/be/src/util/minidump.cc b/be/src/util/minidump.cc
index f722b20..dbaa0cc 100644
--- a/be/src/util/minidump.cc
+++ b/be/src/util/minidump.cc
@@ -116,9 +116,7 @@ static void SetupSigUSR1Handler(bool minidumps_enabled) {
sigaction(SIGUSR1, &sig_action, NULL);
}
-/// Check the number of minidump files and removes the oldest ones to maintain an upper
-/// bound on the number of files.
-static void CheckAndRemoveMinidumps(int max_minidumps) {
+void CheckAndRotateMinidumps(int max_minidumps) {
// Disable rotation if 0 or wrong input
if (max_minidumps <= 0) return;
@@ -225,11 +223,6 @@ Status RegisterMinidump(const char* cmd_line_path) {
return Status(ss.str());
}
- // Rotate old minidump files. We only need to do this on startup (in contrast to
- // periodically) because only process crashes will trigger the creation of new minidump
- // files.
- CheckAndRemoveMinidumps(FLAGS_max_minidumps);
-
google_breakpad::MinidumpDescriptor desc(FLAGS_minidump_path.c_str());
// Limit filesize if configured.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/294d42ad/be/src/util/minidump.h
----------------------------------------------------------------------
diff --git a/be/src/util/minidump.h b/be/src/util/minidump.h
index b2c7160..158e43e 100644
--- a/be/src/util/minidump.h
+++ b/be/src/util/minidump.h
@@ -30,6 +30,10 @@ Status RegisterMinidump(const char* cmd_line_path);
/// tests that deliberately trigger DCHECKs. Returns true if minidumps were previously
/// enabled or false otherwise.
bool EnableMinidumpsForTest(bool enabled);
+
+/// Checks the number of minidump files and removes the oldest ones to maintain an upper
+/// bound on the number of files.
+void CheckAndRotateMinidumps(int max_minidumps);
}
#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/294d42ad/tests/custom_cluster/test_breakpad.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py
index 79ba70c..424e1c0 100644
--- a/tests/custom_cluster/test_breakpad.py
+++ b/tests/custom_cluster/test_breakpad.py
@@ -117,7 +117,7 @@ class TestBreakpad(CustomClusterTestSuite):
return self.cluster.statestored and 1 or 0
raise RuntimeError("Unknown daemon name: %s" % daemon)
- def wait_for_num_processes(self, daemon, num_expected, timeout=60):
+ def wait_for_num_processes(self, daemon, num_expected, timeout=5):
end = time.time() + timeout
self.cluster.refresh()
num_processes = self.get_num_processes(daemon)
@@ -146,7 +146,6 @@ class TestBreakpad(CustomClusterTestSuite):
assert self.count_minidumps('statestored', base_dir) == 1
assert self.count_minidumps('catalogd', base_dir) == 1
-
def assert_num_logfile_entries(self, expected_count):
self.assert_impalad_log_contains('INFO', 'Wrote minidump to ',
expected_count=expected_count)
@@ -172,9 +171,9 @@ class TestBreakpad(CustomClusterTestSuite):
cluster_size = self.get_num_processes('impalad')
self.kill_cluster(SIGUSR1)
# Breakpad forks to write its minidump files, wait for all the clones to terminate.
- assert self.wait_for_num_processes('impalad', cluster_size, 5) == cluster_size
- assert self.wait_for_num_processes('catalogd', 1, 5) == 1
- assert self.wait_for_num_processes('statestored', 1, 5) == 1
+ assert self.wait_for_num_processes('impalad', cluster_size) == cluster_size
+ assert self.wait_for_num_processes('catalogd', 1) == 1
+ assert self.wait_for_num_processes('statestored', 1) == 1
# Make sure impalad still answers queries.
client = self.create_impala_client()
self.execute_query_expect_success(client, "SELECT COUNT(*) FROM functional.alltypes")
@@ -229,6 +228,33 @@ class TestBreakpad(CustomClusterTestSuite):
assert self.count_minidumps('catalogd') == 1
@pytest.mark.execute_serially
+ def test_minidump_cleanup_thread(self):
+ """Check that periodic rotation preserves a limited number of minidumps."""
+ assert self.count_all_minidumps() == 0
+ # Maximum number of minidump that the impalads should keep for this test.
+ max_minidumps = 2
+ # Sleep interval for the log rotation thread.
+ rotation_interval = 1
+ self.start_cluster_with_args(minidump_path=self.tmp_dir,
+ max_minidumps=max_minidumps,
+ logbufsecs=rotation_interval)
+ cluster_size = self.get_num_processes('impalad')
+ # We trigger several rounds of minidump creation to make sure that all daemons wrote
+ # enough files to trigger rotation.
+ for i in xrange(max_minidumps + 1):
+ self.kill_cluster(SIGUSR1)
+ # Breakpad forks to write its minidump files, wait for all the clones to terminate.
+ assert self.wait_for_num_processes('impalad', cluster_size) == cluster_size
+ assert self.wait_for_num_processes('catalogd', 1) == 1
+ assert self.wait_for_num_processes('statestored', 1) == 1
+ self.assert_num_logfile_entries(i + 1)
+ # Sleep long enough for log cleaning to take effect.
+ time.sleep(rotation_interval + 1)
+ assert self.count_minidumps('impalad') == min(cluster_size, max_minidumps)
+ assert self.count_minidumps('statestored') == max_minidumps
+ assert self.count_minidumps('catalogd') == max_minidumps
+
+ @pytest.mark.execute_serially
def test_disable_minidumps(self):
"""Check that setting enable_minidumps to false disables minidump creation."""
assert self.count_all_minidumps() == 0