You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/08/31 05:32:23 UTC
[4/7] incubator-impala git commit: IMPALA-5830:
SET_DENY_RESERVATION_PROBABILITY test
IMPALA-5830: SET_DENY_RESERVATION_PROBABILITY test
Add a targeted test that confirms that setting the query option will
force spilling.
Testing:
Ran test_spilling locally.
Change-Id: Ida6b55b2dee0779b1739af5d75943518ec40d6ce
Reviewed-on: http://gerrit.cloudera.org:8080/7809
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/ebe8ddd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/ebe8ddd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/ebe8ddd4
Branch: refs/heads/release-2.10.0
Commit: ebe8ddd451b3f14d3f778339978a76bcd14b2589
Parents: 1faf89f
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Thu Aug 24 13:26:46 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:54:49 2017 -0700
----------------------------------------------------------------------
.../QueryTest/disable-unsafe-spills.test | 11 ----
.../QueryTest/spilling-query-options.test | 54 ++++++++++++++++++++
tests/common/impala_test_suite.py | 2 +-
tests/query_test/test_spilling.py | 41 +++++++++++----
4 files changed, 85 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
deleted file mode 100644
index a9b8f35..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
+++ /dev/null
@@ -1,11 +0,0 @@
-====
----- QUERY
-# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should
-# not spill to disk.
-set disable_unsafe_spills=true;
-set buffer_pool_limit=40m;
-select distinct *
-from tpch_avro.orders
----- CATCH
-Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false
-====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
new file mode 100644
index 0000000..5b9a641
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
@@ -0,0 +1,54 @@
+====
+---- QUERY
+# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should
+# not spill to disk.
+set disable_unsafe_spills=true;
+set buffer_pool_limit=40m;
+select distinct *
+from tpch_avro.orders
+---- CATCH
+Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on PREPARE.
+set debug_action="-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+ select distinct o_orderdate, o_custkey, o_comment
+ from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on OPEN.
+set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+ select distinct o_orderdate, o_custkey, o_comment
+ from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on GETNEXT.
+# This won't affect the merge aggregation, which accumulates its memory in Open(), but
+# will affect the streaming aggregation.
+set debug_action="-1:GETNEXT:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+ select distinct o_orderdate, o_custkey, o_comment
+ from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index d5841b2..1b7d043 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -210,7 +210,7 @@ class ImpalaTestSuite(BaseTestSuite):
if not query_option in self.default_query_options:
continue
default_val = self.default_query_options[query_option]
- query_str = 'SET '+ query_option + '=' + default_val + ';'
+ query_str = 'SET '+ query_option + '="' + default_val + '"'
try:
impalad_client.execute(query_str)
except Exception as e:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/tests/query_test/test_spilling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py
index c818b65..0b36429 100644
--- a/tests/query_test/test_spilling.py
+++ b/tests/query_test/test_spilling.py
@@ -30,14 +30,14 @@ DEBUG_ACTION_DIMS = [None,
@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
reason='Queries may not spill on larger clusters')
-class TestSpilling(ImpalaTestSuite):
+class TestSpillingDebugActionDimensions(ImpalaTestSuite):
@classmethod
def get_workload(self):
return 'functional-query'
@classmethod
def add_test_dimensions(cls):
- super(TestSpilling, cls).add_test_dimensions()
+ super(TestSpillingDebugActionDimensions, cls).add_test_dimensions()
cls.ImpalaTestMatrix.clear_constraints()
cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
# Tests are calibrated so that they can execute and spill with this page size.
@@ -60,19 +60,38 @@ class TestSpilling(ImpalaTestSuite):
"""Test spilling null-aware anti-joins"""
self.run_test_case('QueryTest/spilling-naaj', vector)
+ def test_spilling_sorts_exhaustive(self, vector):
+ if self.exploration_strategy() != 'exhaustive':
+ pytest.skip("only run large sorts on exhaustive")
+ self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)
+
+
+@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
+ reason='Queries may not spill on larger clusters')
+class TestSpillingNoDebugActionDimensions(ImpalaTestSuite):
+ """Spilling tests to which we don't want to apply the debug_action dimension."""
+ @classmethod
+ def get_workload(self):
+ return 'functional-query'
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestSpillingNoDebugActionDimensions, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.clear_constraints()
+ cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
+ # Tests are calibrated so that they can execute and spill with this page size.
+ cls.ImpalaTestMatrix.add_dimension(
+ create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k']}))
+
def test_spilling_naaj_no_deny_reservation(self, vector):
"""
Null-aware anti-join tests that depend on getting more than the minimum reservation
and therefore will not reliably pass with the deny reservation debug action enabled.
"""
- if vector.get_value('exec_option')['debug_action'] is None:
- self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector)
+ self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector)
- def test_spilling_sorts_exhaustive(self, vector):
- if self.exploration_strategy() != 'exhaustive':
- pytest.skip("only run large sorts on exhaustive")
- self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)
+ def test_spilling_query_options(self, vector):
+ """Test that spilling-related query options work end-to-end. These tests rely on
+ setting debug_action to alternative values via query options."""
+ self.run_test_case('QueryTest/spilling-query-options', vector)
- def test_disable_unsafe_spills(self, vector):
- """Test that the disable_unsafe_spills query options works end-to-end."""
- self.run_test_case('QueryTest/disable-unsafe-spills', vector)