You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by to...@apache.org on 2018/06/19 17:08:39 UTC
[2/3] impala git commit: IMPALA-7115: set a default
THREAD_RESERVATION_LIMIT value
IMPALA-7115: set a default THREAD_RESERVATION_LIMIT value
The value is chosen to allow only queries that have a reasonable chance
of succeeding, albeit with poor performance because of the high
number of threads.
Testing:
Added a test to make sure that the default value rejects a large query.
Change-Id: I31d3fa3f6305c360922649dba53a9026c9563384
Reviewed-on: http://gerrit.cloudera.org:8080/10628
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/894ab8e9
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/894ab8e9
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/894ab8e9
Branch: refs/heads/master
Commit: 894ab8e980ae0e101633680225bafcd34f383223
Parents: 9e069d9
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Wed Jun 6 15:51:37 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Jun 19 03:02:49 2018 +0000
----------------------------------------------------------------------
common/thrift/ImpalaInternalService.thrift | 5 ++++-
.../queries/QueryTest/thread-limits.test | 20 ++++++++++++++++++++
2 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/894ab8e9/common/thrift/ImpalaInternalService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift
index 96314d5..f3d8af3 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++ b/common/thrift/ImpalaInternalService.thrift
@@ -291,7 +291,10 @@ struct TQueryOptions {
65: optional i64 max_mem_estimate_for_admission = 0;
// See comment in ImpalaService.thrift.
- 66: optional i32 thread_reservation_limit = 0;
+ // The default values is set fairly high based on empirical data - queries with up to
+ // this number of reserved threads have run successfully as part of production
+ // workloads but with very degraded performance.
+ 66: optional i32 thread_reservation_limit = 3000;
// See comment in ImpalaService.thrift.
67: optional i32 thread_reservation_aggregate_limit = 0;
http://git-wip-us.apache.org/repos/asf/impala/blob/894ab8e9/testdata/workloads/functional-query/queries/QueryTest/thread-limits.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/thread-limits.test b/testdata/workloads/functional-query/queries/QueryTest/thread-limits.test
index ad67c70..1730d38 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/thread-limits.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/thread-limits.test
@@ -102,3 +102,23 @@ BIGINT
---- RESULTS
7300
====
+---- QUERY
+# Default value should prevent crazy queries from running.
+# The below query should have > 3000 required threads per backend.
+set mt_dop=3;
+with
+ c1 as (select count(*) from functional.alltypes),
+ c2 as (select * from c1 union select * from c1),
+ c4 as (select * from c2 union select * from c2),
+ c8 as (select * from c4 union select * from c4),
+ c16 as (select * from c8 union select * from c8),
+ c32 as (select * from c16 union select * from c16),
+ c64 as (select * from c32 union select * from c32),
+ c128 as (select * from c64 union select * from c64),
+ c256 as (select * from c128 union select * from c128),
+ c512 as (select * from c256 union select * from c256)
+select * from c512 union select * from c512;
+---- CATCH
+row_regex:.*Rejected query from pool default-pool: thread reservation on backend '.*'
+ is greater than the THREAD_RESERVATION_LIMIT query option value: [0-9]* > 3000\.
+====