You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/08/21 21:38:53 UTC
[3/3] impala git commit: IMPALA-7465: fix test_kudu_scan_mem_usage
IMPALA-7465: fix test_kudu_scan_mem_usage
The issue was that the row batch queue could grow a lot if the consumer
was slow.
Also add an additional test to exercise the OOM code path in Kudu for
completeness.
Testing:
Added sleep to kudu-scan-node.cc that reproduced the problem.
Looped modified test to flush out flakiness.
Change-Id: Ic4a95b6b6d96a447df68ef4912a86f1e11f219ca
Reviewed-on: http://gerrit.cloudera.org:8080/11285
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/bddd7def
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/bddd7def
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/bddd7def
Branch: refs/heads/master
Commit: bddd7def991f60f87772299df1aa53bc0fde48f2
Parents: 2a60655
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Aug 21 10:53:47 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Aug 21 21:38:11 2018 +0000
----------------------------------------------------------------------
.../queries/QueryTest/kudu-scan-mem-usage.test | 11 +++++++++++
tests/query_test/test_mem_usage_scaling.py | 2 ++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/bddd7def/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-mem-usage.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-mem-usage.test b/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-mem-usage.test
index aa42edb..7fd6e3c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-mem-usage.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-mem-usage.test
@@ -2,6 +2,8 @@
---- QUERY
# IMPALA-7076: this query will fail if the Kudu scan spins up too many scanner threads.
# Only one scanner thread per impalad should be started.
+# Set num_scanner_threads to reduce the row batch queue length and avoid IMPALA-7465.
+set num_scanner_threads=2;
set mem_limit=4mb;
select * from tpch_kudu.orders
order by o_orderkey limit 3;
@@ -14,3 +16,12 @@ BIGINT,BIGINT,STRING,DECIMAL,STRING,STRING,STRING,INT,STRING
---- RUNTIME_PROFILE
aggregation(SUM, NumScannerThreadsStarted): 3
====
+---- QUERY
+# Test with lower limit that reliably hits memory limit exceeded.
+set disable_codegen=true;
+set mem_limit=32k;
+select * from tpch_kudu.orders
+order by o_orderkey limit 3;
+---- CATCH
+Memory limit exceeded
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/bddd7def/tests/query_test/test_mem_usage_scaling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py
index 0d3219a..21b320e 100644
--- a/tests/query_test/test_mem_usage_scaling.py
+++ b/tests/query_test/test_mem_usage_scaling.py
@@ -353,6 +353,8 @@ class TestScanMemLimit(ImpalaTestSuite):
def test_kudu_scan_mem_usage(self, vector):
"""Test that Kudu scans can stay within a low memory limit. Before IMPALA-7096 they
were not aware of mem_limit and would start up too many scanner threads."""
+ # .test file overrides disable_codegen.
+ del vector.get_value('exec_option')['disable_codegen']
self.run_test_case('QueryTest/kudu-scan-mem-usage', vector)
def test_hdfs_scanner_thread_mem_scaling(self, vector):