You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/04/14 15:28:29 UTC
(impala) 01/02: IMPALA-14840: Enforce pool Max Memory when query limits are unset
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6cbab68914cf536ade27db635f8aa34f2cb43428
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Thu Mar 19 18:02:40 2026 +0100
IMPALA-14840: Enforce pool Max Memory when query limits are unset
Impala should use memory-based query admission when Max Memory is set
for the given pool. However, if Min Query Memory Limit, Max Query Memory
Limit are missing, and query option MEM_LIMIT is not set, then currently
queries can run with unlimited resources.
This can cause a situation when Max Memory is specified, but the pool
can consume basically unlimited memory.
This patch fixes ScheduleState::UpdateMemoryRequirements() to not turn
off memory-based query admission when Max Memory is set.
The patch also removes the unused 'root_cfg' parameter of
AdmissionController::CanAdmitRequest().
Testing
* new admission controller backend tests added
* updated pool 'poolNoMemLimits' in mem-limit-test-fair-scheduler.xml
to not specify Max Memory
Change-Id: I4834964e4361895e10627a661831253ce676c129
Reviewed-on: http://gerrit.cloudera.org:8080/24129
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/scheduling/admission-controller-test.cc | 137 +++++++++++++++++----
be/src/scheduling/admission-controller.cc | 7 +-
be/src/scheduling/admission-controller.h | 11 +-
be/src/scheduling/schedule-state.cc | 11 +-
.../java/org/apache/impala/analysis/Analyzer.java | 11 ++
.../resources/mem-limit-test-fair-scheduler.xml | 1 -
tests/custom_cluster/test_admission_controller.py | 3 +-
7 files changed, 144 insertions(+), 37 deletions(-)
diff --git a/be/src/scheduling/admission-controller-test.cc b/be/src/scheduling/admission-controller-test.cc
index 694e6d341..f2e7a5a0c 100644
--- a/be/src/scheduling/admission-controller-test.cc
+++ b/be/src/scheduling/admission-controller-test.cc
@@ -562,8 +562,12 @@ class AdmissionControllerTest : public testing::Test {
string not_admitted_reason = "--not set--";
const bool mimic_old_behaviour = test_pool_config.min_query_mem_limit == 0
&& test_pool_config.max_query_mem_limit == 0;
+ // backend_mem_unlimited is true only when there are truly no memory constraints:
+ // no per-query limits from pool config, no query options, and the pool itself does
+ // not have a max_mem_resources cap.
const bool backend_mem_unlimited = mimic_old_behaviour && mem_limit < 0
- && mem_limit_executors < 0 && mem_limit_coordinators < 0;
+ && mem_limit_executors < 0 && mem_limit_coordinators < 0
+ && test_pool_config.max_mem_resources <= 0;
if (backend_mem_unlimited) {
ASSERT_EQ(-1, test_state->per_backend_mem_limit());
@@ -653,7 +657,7 @@ TEST_F(AdmissionControllerTest, Simple) {
string not_admitted_reason;
bool coordinator_resource_limited = false;
ASSERT_TRUE(admission_controller->CanAdmitRequest(*schedule_state, config_c,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
ASSERT_FALSE(coordinator_resource_limited);
ASSERT_EQ(0, admission_controller->root_agg_user_loads_.size());
@@ -662,7 +666,7 @@ TEST_F(AdmissionControllerTest, Simple) {
ScheduleState* schedule_state_3_hosts =
MakeScheduleState(QUEUE_C, config_c, 3, 64L * MEGABYTE);
ASSERT_FALSE(admission_controller->CanAdmitRequest(*schedule_state_3_hosts, config_c,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(not_admitted_reason,
"Not enough aggregate memory available in pool root.queueC with max mem "
"resources 128.00 MB. Needed 192.00 MB but only 128.00 MB was available.");
@@ -706,7 +710,7 @@ TEST_F(AdmissionControllerTest, Simple) {
// Test that the query cannot be admitted now.
ASSERT_FALSE(admission_controller->CanAdmitRequest(*schedule_state, config_c,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(
not_admitted_reason, "number of running queries 11 is at or over limit 10.");
ASSERT_FALSE(coordinator_resource_limited);
@@ -824,7 +828,7 @@ TEST_F(AdmissionControllerTest, CanAdmitRequestMemory) {
string not_admitted_reason;
bool coordinator_resource_limited = false;
ASSERT_TRUE(admission_controller->CanAdmitRequest(*schedule_state, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
ASSERT_FALSE(coordinator_resource_limited);
// Tests that this query cannot be admitted.
@@ -834,7 +838,7 @@ TEST_F(AdmissionControllerTest, CanAdmitRequestMemory) {
ScheduleState* schedule_state15 =
MakeScheduleState(QUEUE_D, config_d, host_count, 30L * MEGABYTE);
ASSERT_FALSE(admission_controller->CanAdmitRequest(*schedule_state15, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(not_admitted_reason,
"Not enough aggregate memory available in pool root.queueD with max mem resources "
"400.00 MB. Needed 480.00 MB but only 400.00 MB was available.");
@@ -847,7 +851,7 @@ TEST_F(AdmissionControllerTest, CanAdmitRequestMemory) {
MakeScheduleState(QUEUE_D, config_d, host_count, 50L * MEGABYTE);
ASSERT_FALSE(admission_controller->CanAdmitRequest(*schedule_state_10_fail, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(not_admitted_reason,
"Not enough aggregate memory available in pool root.queueD with max mem resources "
"400.00 MB. Needed 500.00 MB but only 400.00 MB was available.");
@@ -888,11 +892,11 @@ TEST_F(AdmissionControllerTest, CanAdmitRequestCount) {
// Query can be admitted from queue...
bool coordinator_resource_limited = false;
ASSERT_TRUE(admission_controller->CanAdmitRequest(*schedule_state, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
ASSERT_FALSE(coordinator_resource_limited);
// ... but same Query cannot be admitted directly.
ASSERT_FALSE(admission_controller->CanAdmitRequest(*schedule_state, config_d,
- config_root, false, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ false, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(not_admitted_reason,
"queue is not empty (size 2); queued queries are executed first");
ASSERT_FALSE(coordinator_resource_limited);
@@ -900,7 +904,7 @@ TEST_F(AdmissionControllerTest, CanAdmitRequestCount) {
// Simulate that there are 7 queries already running.
pool_stats->agg_num_running_ = 7;
ASSERT_FALSE(admission_controller->CanAdmitRequest(*schedule_state, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(
not_admitted_reason, "number of running queries 7 is at or over limit 6");
ASSERT_FALSE(coordinator_resource_limited);
@@ -945,11 +949,11 @@ TEST_F(AdmissionControllerTest, UserAndGroupQuotas) {
// Query can be admitted from queue...
bool coordinator_resource_limited = false;
ASSERT_TRUE(admission_controller->CanAdmitRequest(*schedule_state, config_e,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
ASSERT_FALSE(coordinator_resource_limited);
// ... but same Query cannot be admitted directly.
ASSERT_FALSE(admission_controller->CanAdmitRequest(*schedule_state, config_e,
- config_root, false, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ false, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(not_admitted_reason,
"queue is not empty (size 2); queued queries are executed first");
ASSERT_FALSE(coordinator_resource_limited);
@@ -957,7 +961,7 @@ TEST_F(AdmissionControllerTest, UserAndGroupQuotas) {
// Simulate that there are 7 queries already running.
pool_stats->agg_num_running_ = 7;
ASSERT_FALSE(admission_controller->CanAdmitRequest(*schedule_state, config_e,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
// Limit of requests is 5 from llama.am.throttling.maximum.placed.reservations.
EXPECT_STR_CONTAINS(
not_admitted_reason, "number of running queries 7 is at or over limit 5");
@@ -965,7 +969,7 @@ TEST_F(AdmissionControllerTest, UserAndGroupQuotas) {
pool_stats->agg_num_running_ = 3;
ASSERT_TRUE(admission_controller->CanAdmitRequest(*schedule_state, config_e,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
// Test CanAdmitQuota
// Make sure queue is empty as we are going to pass admit_from_queue=false.
@@ -1158,11 +1162,11 @@ TEST_F(AdmissionControllerTest, CanAdmitRequestSlots) {
// Enough slots are available, so it can be admitted in both cases.
ASSERT_TRUE(admission_controller->CanAdmitRequest(*default_group_schedule, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
<< not_admitted_reason;
ASSERT_FALSE(coordinator_resource_limited);
ASSERT_TRUE(admission_controller->CanAdmitRequest(*other_group_schedule, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
<< not_admitted_reason;
ASSERT_FALSE(coordinator_resource_limited);
@@ -1171,18 +1175,18 @@ TEST_F(AdmissionControllerTest, CanAdmitRequestSlots) {
SetSlotsInUse(admission_controller, host_addrs, slots_per_host - 1);
ASSERT_TRUE(admission_controller->CanAdmitRequest(*default_group_schedule, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
<< not_admitted_reason;
ASSERT_FALSE(coordinator_resource_limited);
ASSERT_FALSE(admission_controller->CanAdmitRequest(*other_group_schedule, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(not_admitted_reason,
"Not enough admission control slots available on host host1:25000. Needed 4 "
"slots but 15/16 are already in use.");
ASSERT_FALSE(coordinator_resource_limited);
// Assert that coordinator-only schedule also not admitted.
ASSERT_FALSE(admission_controller->CanAdmitRequest(*coordinator_only_schedule, config_d,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
EXPECT_STR_CONTAINS(not_admitted_reason,
"Not enough admission control slots available on host host0:25000. Needed 4 "
"slots but 15/16 are already in use.");
@@ -1231,14 +1235,14 @@ TEST_F(AdmissionControllerTest, CanAdmitRequestSlotsDefault) {
// All schedules should be admitted.
SetSlotsInUse(admission_controller, host_addrs, slots_per_host);
ASSERT_TRUE(admission_controller->CanAdmitRequest(*default_pool_schedule, pool_config,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
<< not_admitted_reason;
ASSERT_FALSE(coordinator_resource_limited);
ASSERT_EQ(coordinator_only_schedule->executor_group(),
ClusterMembershipMgr::EMPTY_GROUP_NAME);
ASSERT_TRUE(
admission_controller->CanAdmitRequest(*coordinator_only_schedule, pool_config,
- config_root, true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited))
<< not_admitted_reason;
ASSERT_FALSE(coordinator_resource_limited);
}
@@ -2087,7 +2091,7 @@ TEST_F(AdmissionControllerTest, TopNQueryCheck) {
string not_admitted_reason;
bool coordinator_resource_limited = false;
ASSERT_TRUE(
- admission_controller->CanAdmitRequest(*schedule_state, pool_config, config_root,
+ admission_controller->CanAdmitRequest(*schedule_state, pool_config,
true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
ASSERT_FALSE(coordinator_resource_limited);
// Create a query memory tracker for the query and set the memory consumption
@@ -2258,4 +2262,93 @@ TEST_F(AdmissionControllerTest, HasSufficientGroupQuota) {
"group limit 4 in pool 'poolname' (Ignored Group Quotas 'group2':1 'group1':3)");
}
+/// Test that when a pool has max_mem_resources set but no min/max_query_mem_limit,
+/// a query with no MEM_LIMIT set is capped by the estimates.
+TEST_F(AdmissionControllerTest, LimitsWhenOnlyMaxMemResourcesSet) {
+ const int64_t pool_max_mem = 2 * GIGABYTE;
+ const int num_backends = 2;
+
+ // Pool has max_mem_resources but no min/max_query_mem_limit (mimic old behaviour).
+ TPoolConfig pool_config;
+ pool_config.__set_min_query_mem_limit(0);
+ pool_config.__set_max_query_mem_limit(0);
+ pool_config.__set_max_mem_resources(pool_max_mem);
+
+ // No MEM_LIMIT set: should use estimates.
+ ScheduleState* schedule_state = MakeScheduleState("default", -1, pool_config,
+ num_backends, DEFAULT_PER_EXEC_MEM_ESTIMATE, DEFAULT_COORD_MEM_ESTIMATE, true);
+ EXPECT_EQ(DEFAULT_PER_EXEC_MEM_ESTIMATE, schedule_state->per_backend_mem_limit());
+ EXPECT_EQ(DEFAULT_COORD_MEM_ESTIMATE, schedule_state->coord_backend_mem_limit());
+ EXPECT_EQ(MemLimitSourcePB::QUERY_PLAN_PER_HOST_MEM_ESTIMATE,
+ schedule_state->per_backend_mem_to_admit_source());
+
+ // When the planner estimate is large, the runtime limit equals the estimate.
+ // Queries exceeding the aggregate pool budget are rejected by the admission
+ // controller; the runtime limit is never artificially capped below the estimate.
+ // 3 GB is > 2 GB / 2 (pool_max_mem / num_backends), ensuring the estimate exceeds
+ // the per-node pool budget.
+ const int64_t large_estimate = 3 * GIGABYTE;
+ ScheduleState* large_state = MakeScheduleState("default", -1, pool_config,
+ num_backends, large_estimate, large_estimate, true);
+ EXPECT_EQ(large_estimate, large_state->per_backend_mem_limit());
+ EXPECT_EQ(large_estimate, large_state->coord_backend_mem_limit());
+ EXPECT_EQ(MemLimitSourcePB::QUERY_PLAN_PER_HOST_MEM_ESTIMATE,
+ large_state->per_backend_mem_to_admit_source());
+
+ // Single backend (coordinator-only query): the coordinator-only optimization sets
+ // per_backend_mem_to_admit=0, so per_backend_mem_limit=0. The coordinator's limit
+ // equals the planner estimate.
+ ScheduleState* single_backend_state = MakeScheduleState("default", -1, pool_config,
+ 1, DEFAULT_PER_EXEC_MEM_ESTIMATE, DEFAULT_COORD_MEM_ESTIMATE, false);
+ EXPECT_EQ(0, single_backend_state->per_backend_mem_limit());
+ EXPECT_EQ(DEFAULT_PER_EXEC_MEM_ESTIMATE,
+ single_backend_state->coord_backend_mem_limit());
+
+ // When max_mem_resources <= 0, the original unlimited (-1) behavior is preserved.
+ TPoolConfig unlimited_pool_config;
+ unlimited_pool_config.__set_min_query_mem_limit(0);
+ unlimited_pool_config.__set_max_query_mem_limit(0);
+ unlimited_pool_config.__set_max_mem_resources(-1); // unlimited pool
+ ScheduleState* unlimited_state = MakeScheduleState("default", -1,
+ unlimited_pool_config, num_backends, DEFAULT_PER_EXEC_MEM_ESTIMATE,
+ DEFAULT_COORD_MEM_ESTIMATE, false);
+ EXPECT_EQ(-1, unlimited_state->per_backend_mem_limit());
+ EXPECT_EQ(-1, unlimited_state->coord_backend_mem_limit());
+}
+
+/// Test that when a pool has max_mem_resources set but no min/max_query_mem_limit,
+/// the runtime mem limit equals the plan estimate (not -1), so reservation checks
+/// and pool memory accounting work correctly.
+TEST_F(AdmissionControllerTest, CanAdmitWhenOnlyMaxMemResourcesSet) {
+ AdmissionController* admission_controller = MakeAdmissionController();
+ TPoolConfig test_pool_config;
+ test_pool_config.__set_min_query_mem_limit(0);
+ test_pool_config.__set_max_query_mem_limit(0);
+ test_pool_config.__set_max_requests(10);
+ test_pool_config.__set_max_mem_resources(2 * GIGABYTE);
+
+ string not_admitted_reason;
+ bool coordinator_resource_limited = false;
+
+ // Initial reservations fit and admission control can admit query to pool.
+ ScheduleState* test_state = MakeScheduleState("default", -1, test_pool_config, 3,
+ 800 * MEGABYTE, 300 * MEGABYTE, true);
+ // 2 * 800 MB + 300 MB = 1.9 GB < 2 GB max_mem_resources, so admission succeeds.
+ admission_controller->CanAdmitRequest(*test_state, test_pool_config,
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited);
+ EXPECT_EQ("", not_admitted_reason);
+ ASSERT_FALSE(coordinator_resource_limited);
+
+ // Large query: Initial mem reservation checks pass but aggregate cluster estimate
+ // exceeds pool max_mem_resources.
+ not_admitted_reason.clear();
+ test_state = MakeScheduleState("default", -1, test_pool_config, 3,
+ 800 * MEGABYTE, 500 * MEGABYTE, true);
+ // 2 * 800 MB + 500 MB = 2.1 GB > 2 GB max_mem_resources, so admission fails.
+ ASSERT_FALSE(admission_controller->CanAdmitRequest(*test_state, test_pool_config,
+ true, ¬_admitted_reason, nullptr, coordinator_resource_limited));
+ EXPECT_STR_CONTAINS(not_admitted_reason, "Not enough aggregate memory available");
+ ASSERT_FALSE(coordinator_resource_limited);
+}
+
} // end namespace impala
diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc
index 4f8b0dc01..5fd0ac5f4 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -1382,9 +1382,8 @@ bool AdmissionController::CanAdmitTrivialRequest(const ScheduleState& state) {
}
bool AdmissionController::CanAdmitRequest(const ScheduleState& state,
- const TPoolConfig& pool_cfg, const TPoolConfig& root_cfg, bool admit_from_queue,
- string* not_admitted_reason, string* not_admitted_details,
- bool& coordinator_resource_limited) {
+ const TPoolConfig& pool_cfg, bool admit_from_queue, string* not_admitted_reason,
+ string* not_admitted_details, bool& coordinator_resource_limited) {
// Can't admit if:
// (a) There are already queued requests (and this is not admitting from the queue).
// (b) The resource pool is already at the maximum number of requests.
@@ -2552,7 +2551,7 @@ bool AdmissionController::FindGroupToAdmitOrReject(
queue_node->not_admitted_reason = rejection_reason;
return false;
}
- if (CanAdmitRequest(*state, pool_config, root_cfg, admit_from_queue,
+ if (CanAdmitRequest(*state, pool_config, admit_from_queue,
&queue_node->not_admitted_reason, &queue_node->not_admitted_details,
coordinator_resource_limited)) {
queue_node->admitted_schedule = std::move(group_state.state);
diff --git a/be/src/scheduling/admission-controller.h b/be/src/scheduling/admission-controller.h
index 9a272acd5..8604452c1 100644
--- a/be/src/scheduling/admission-controller.h
+++ b/be/src/scheduling/admission-controller.h
@@ -190,10 +190,10 @@ class AdmissionExecRequestCompressed : public AdmissionExecRequest {
/// memory limit and a lower bound is enforced on it based on the largest initial
/// reservation of the query. The final memory limit used is also clamped by the max/min
/// memory limits configured for the pool with an option to not enforce these limits on
-/// the MEM_LIMIT query option (If both these max/min limits are not configured, then the
-/// estimates from planning are not used as a memory limit and are only used for making
-/// admission decisions. Moreover the estimates will no longer have a lower bound based on
-/// the largest initial reservation).
+/// the MEM_LIMIT query option (If no max_mem_resources, and no min/max limits are
+/// configured for the pool, then the estimates from planning are not used as a memory
+/// limit and are only used for making admission decisions. Moreover, the estimates will
+/// no longer have a lower bound based on the largest initial reservation).
/// The following four conditions must hold in order for the request to be admitted:
/// 1) The current pool configuration is valid.
/// 2) There must be enough memory resources available in this resource pool for the
@@ -1215,7 +1215,7 @@ class AdmissionController {
/// enough memory resources available for the query. Caller owns not_admitted_reason and
/// not_admitted_details. Must hold admission_ctrl_lock_.
bool CanAdmitRequest(const ScheduleState& state, const TPoolConfig& pool_cfg,
- const TPoolConfig& root_cfg, bool admit_from_queue, string* not_admitted_reason,
+ bool admit_from_queue, string* not_admitted_reason,
string* not_admitted_details, bool& coordinator_resource_limited);
/// Returns true if User Quotas allow the schedule to be admitted to the pool with
@@ -1485,6 +1485,7 @@ class AdmissionController {
FRIEND_TEST(AdmissionControllerTest, CanAdmitRequestMemory);
FRIEND_TEST(AdmissionControllerTest, CanAdmitRequestSlots);
FRIEND_TEST(AdmissionControllerTest, CanAdmitRequestSlotsDefault);
+ FRIEND_TEST(AdmissionControllerTest, CanAdmitWhenOnlyMaxMemResourcesSet);
FRIEND_TEST(AdmissionControllerTest, DedicatedCoordAdmissionChecks);
FRIEND_TEST(AdmissionControllerTest, DedicatedCoordScheduleState);
FRIEND_TEST(AdmissionControllerTest, DequeueLoop);
diff --git a/be/src/scheduling/schedule-state.cc b/be/src/scheduling/schedule-state.cc
index dd878f89a..3c6d8a9cb 100644
--- a/be/src/scheduling/schedule-state.cc
+++ b/be/src/scheduling/schedule-state.cc
@@ -298,11 +298,14 @@ void ScheduleState::CompareMinCoordinatorMemToAdmit(
// Please update Analyzer.getMaxMemLimitPerHost() in Analyzer.java if updating this.
void ScheduleState::UpdateMemoryRequirements(const TPoolConfig& pool_cfg,
int64_t coord_mem_limit_admission, int64_t executor_mem_limit_admission) {
- // If the min_query_mem_limit and max_query_mem_limit are not set in the pool config
- // then it falls back to traditional(old) behavior, which means that, it sets the
- // mem_limit if it is set in the query options, else sets it to -1 (no limit).
+ // If the max_mem_resources, min_query_mem_limit, and max_query_mem_limit are not set
+ // in the pool config then it falls back to traditional(old) behavior, which means
+ // that, it sets the mem_limit if it is set in the query options, else sets it to
+ // -1 (no limit).
const bool mimic_old_behaviour =
- pool_cfg.min_query_mem_limit == 0 && pool_cfg.max_query_mem_limit == 0;
+ pool_cfg.max_mem_resources <= 0 &&
+ pool_cfg.min_query_mem_limit == 0 &&
+ pool_cfg.max_query_mem_limit == 0;
const bool use_dedicated_coord_estimates = UseDedicatedCoordEstimates();
query_schedule_pb_->set_per_backend_mem_to_admit(0);
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 4109ea174..a8357e972 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -644,6 +644,9 @@ public class Analyzer {
// which is the maximum memory limit of destination request pool for this query in
// bytes. Value 0 or less means unknown.
private long poolMaxMemLimit_ = -1;
+ // Value of impala.admission-control.max-mem-resources.<pool_name>,
+ // which is the total memory cap for the pool across all backends, in bytes.
+ private long poolMaxMemResources_ = -1;
// Cache of KuduTables opened for this query. (map from table name to kudu table)
// This cache prevent multiple openTable calls for a given table in the same query.
@@ -799,6 +802,11 @@ public class Analyzer {
// No MEM_LIMIT* option is set and max-query-mem-limit is set.
// Return the max-query-mem-limit regardless of clamp-mem-limit-query-option value.
return globalState_.poolMaxMemLimit_;
+ } else if (globalState_.poolMaxMemResources_ > 0) {
+ // No MEM_LIMIT* option is set and max-query-mem-limit is not set, but
+ // max-mem-resources is set. So return the maximum memory that can be admitted to
+ // this pool per executor.
+ return globalState_.poolMaxMemResources_ / Math.max(numExecutorsForPlanning(), 1);
} else if (opts.isSetMax_mem_estimate_for_admission()
&& opts.getMax_mem_estimate_for_admission() > 0) {
// MAX_MEM_ESTIMATE_FOR_ADMISSION
@@ -829,6 +837,9 @@ public class Analyzer {
if (poolConfig.isSetMax_query_mem_limit()) {
globalState_.poolMaxMemLimit_ = poolConfig.getMax_query_mem_limit();
}
+ if (poolConfig.isSetMax_mem_resources() && poolConfig.getMax_mem_resources() > 0) {
+ globalState_.poolMaxMemResources_ = poolConfig.getMax_mem_resources();
+ }
}
}
diff --git a/fe/src/test/resources/mem-limit-test-fair-scheduler.xml b/fe/src/test/resources/mem-limit-test-fair-scheduler.xml
index 347164297..9c2e26b61 100644
--- a/fe/src/test/resources/mem-limit-test-fair-scheduler.xml
+++ b/fe/src/test/resources/mem-limit-test-fair-scheduler.xml
@@ -23,7 +23,6 @@
</queue>
<queue name="poolNoMemLimits">
<aclSubmitApps>*</aclSubmitApps>
- <maxResources>2500 mb, 0 vcores</maxResources>
</queue>
<queue name="maxLessThanMinLimit">
<aclSubmitApps>*</aclSubmitApps>
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 387d93eb4..939e6def4 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -566,8 +566,9 @@ class TestAdmissionController(TestAdmissionControllerBase):
from functional_parquet.alltypes""".format(target_tbl),
"Rejected query from pool")
insert_fail = unique_database + ".insert_fail"
+ # Use DDL to create the table so it is not subject to admission control.
self.execute_query_expect_success(self.client,
- """create table {0} as select 100000""".format(insert_fail))
+ """create table {0} (id int)""".format(insert_fail))
verify_json_plan("""insert into {0} select id from functional_parquet.alltypes"""
.format(insert_fail),
"Rejected query from pool")