You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/08/24 06:31:25 UTC
[2/2] incubator-impala git commit: IMPALA-5780,
IMPALA-5779: extra spilling tests
IMPALA-5780,IMPALA-5779: extra spilling tests
* Test for disable_unsafe_spills
* Test for buffer size > I/O size (--read_size)
Change-Id: I03de00394bb6bbcf381250f816e22a4b987f1135
Reviewed-on: http://gerrit.cloudera.org:8080/7787
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/bb6b0ce2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/bb6b0ce2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/bb6b0ce2
Branch: refs/heads/master
Commit: bb6b0ce249f561af04ea42752ca874cd93dd63c9
Parents: 88d00b4
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Wed Aug 23 13:47:37 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Aug 24 06:27:05 2017 +0000
----------------------------------------------------------------------
.../QueryTest/disable-unsafe-spills.test | 11 +++++++
.../QueryTest/spilling-sorts-exhaustive.test | 31 +++++++++++++++++++-
tests/query_test/test_spilling.py | 4 +++
3 files changed, 45 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb6b0ce2/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
new file mode 100644
index 0000000..a9b8f35
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
@@ -0,0 +1,11 @@
+====
+---- QUERY
+# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should
+# not spill to disk.
+set disable_unsafe_spills=true;
+set buffer_pool_limit=40m;
+select distinct *
+from tpch_avro.orders
+---- CATCH
+Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb6b0ce2/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
index d391884..415fe5d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
@@ -192,4 +192,33 @@ STRING
# Verify that the sort actually spilled
row_regex: .*SpilledRuns: .* \([1-9][0-9]*\)
row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
-
+====
+---- QUERY
+# Test that we can spill with a buffer size larger than the default I/O size (8mb).
+set default_spillable_buffer_size=16m;
+set buffer_pool_limit=96m;
+select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
+from tpch_parquet.lineitem
+order by 1, 2, 3, 4, 5
+offset 6001200
+---- TYPES
+BIGINT,BIGINT,BIGINT,INT,STRING
+---- RESULTS
+5999971,97922,7923,5,'place of the slyly quick pla'
+5999971,132903,2904,4,'ckly above the boldly '
+5999971,144823,4824,2,'e blithely after the carefully pending '
+5999971,161882,6915,6,'luffy theodolites nag boldly bli'
+5999972,102942,473,1,'s maintain carefully among the'
+5999972,133109,8136,2,' the furiously express pearls. furi'
+5999972,152761,2762,3,'sual accounts al'
+5999973,176345,1380,1,'gular excuses. '
+5999974,10463,5466,2,'dolites wake'
+5999974,25360,5361,1,'express dependencies. express, pendi'
+5999975,6452,1453,2,'lar pinto beans aft'
+5999975,7272,2273,1,'tructions. excu'
+5999975,37131,2138,3,', quick deposits. ironic, unusual deposi'
+6000000,32255,2256,1,'carefully '
+6000000,96127,6128,2,'ooze furiously about the pe'
+---- RUNTIME_PROFILE
+row_regex: .*SpilledRuns: .* \([1-9][0-9]*\)
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb6b0ce2/tests/query_test/test_spilling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py
index 437739c..c818b65 100644
--- a/tests/query_test/test_spilling.py
+++ b/tests/query_test/test_spilling.py
@@ -72,3 +72,7 @@ class TestSpilling(ImpalaTestSuite):
if self.exploration_strategy() != 'exhaustive':
pytest.skip("only run large sorts on exhaustive")
self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)
+
+ def test_disable_unsafe_spills(self, vector):
+ """Test that the disable_unsafe_spills query options works end-to-end."""
+ self.run_test_case('QueryTest/disable-unsafe-spills', vector)