You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by cs...@apache.org on 2019/09/11 13:06:43 UTC
[impala] 02/03: IMPALA-8936: Improve queuing reason for unhealthy
executor groups
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 576f205bffadd0a18cbc36167e6f191279fbc911
Author: Lars Volker <lv...@cloudera.com>
AuthorDate: Tue Sep 10 20:13:12 2019 -0700
IMPALA-8936: Improve queuing reason for unhealthy executor groups
In some situations, users might actually expect not having a healthy
executor group around, e.g. when they're starting one and it takes a
while to come online. This change makes the queuing reason more generic
and drops the "unhealthy" concept from it to reduce confusion.
Change-Id: Idceab7fb56335bab9d787b0f351a41e6efd7dd59
Reviewed-on: http://gerrit.cloudera.org:8080/14210
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/scheduling/admission-controller.cc | 5 +++--
tests/custom_cluster/test_coordinators.py | 4 ++--
tests/custom_cluster/test_executor_groups.py | 11 +++++------
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc
index ff0ea4c..164fed2 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -214,7 +214,8 @@ const string REASON_THREAD_RESERVATION_AGG_LIMIT_EXCEEDED =
// $0 is the error message returned by the scheduler.
const string REASON_SCHEDULER_ERROR = "Error during scheduling: $0";
const string REASON_LOCAL_BACKEND_NOT_STARTED = "Local backend has not started up yet.";
-const string REASON_NO_EXECUTOR_GROUPS = "No healthy executor groups found for pool $0.";
+const string REASON_NO_EXECUTOR_GROUPS = "Waiting for executors to start. Only DDL "
+ "queries can currently run.";
// Queue decision details
// $0 = num running queries, $1 = num queries limit, $2 = num queries limit explanation,
@@ -1193,7 +1194,7 @@ Status AdmissionController::ComputeGroupSchedules(
membership_snapshot->executor_groups, pool_name, &executor_groups);
if (executor_groups.empty()) {
- queue_node->not_admitted_reason = Substitute(REASON_NO_EXECUTOR_GROUPS, pool_name);
+ queue_node->not_admitted_reason = REASON_NO_EXECUTOR_GROUPS;
LOG(WARNING) << queue_node->not_admitted_reason;
return Status::OK();
}
diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py
index c6f92f9..997e7ed 100644
--- a/tests/custom_cluster/test_coordinators.py
+++ b/tests/custom_cluster/test_coordinators.py
@@ -295,6 +295,6 @@ class TestCoordinators(CustomClusterTestSuite):
present."""
result = self.execute_query_expect_failure(self.client, "select 2")
expected_error = "Query aborted:Admission for query exceeded timeout 60000ms in " \
- "pool default-pool. Queued reason: No healthy executor groups " \
- "found for pool default-pool."
+ "pool default-pool. Queued reason: Waiting for executors to " \
+ "start. Only DDL queries can currently run."
assert expected_error in str(result)
diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py
index 32e0087..1e8ba39 100644
--- a/tests/custom_cluster/test_executor_groups.py
+++ b/tests/custom_cluster/test_executor_groups.py
@@ -109,7 +109,7 @@ class TestExecutorGroups(CustomClusterTestSuite):
client = self.client
handle = client.execute_async(QUERY)
profile = client.get_runtime_profile(handle)
- assert "No healthy executor groups found for pool" in profile
+ assert "Waiting for executors to start" in profile
assert self.coordinator.service.get_metric_value(
"cluster-membership.executor-groups.total-healthy") == 0
self._add_executor_group("group1", 2)
@@ -138,7 +138,7 @@ class TestExecutorGroups(CustomClusterTestSuite):
# Run query and observe timeout
handle = client.execute_async(QUERY)
profile = client.get_runtime_profile(handle)
- assert "No healthy executor groups found for pool" in profile, profile
+ assert "Waiting for executors to start" in profile, profile
# Restart executor
executor.start()
# Query should now finish
@@ -286,8 +286,7 @@ class TestExecutorGroups(CustomClusterTestSuite):
client = self.client
handle = client.execute_async(QUERY)
profile = client.get_runtime_profile(handle)
- assert "Initial admission queue reason: No healthy executor groups found for pool" \
- in profile
+ assert "Initial admission queue reason: Waiting for executors to start" in profile
initial_state = client.get_state(handle)
# Start another executor and observe that the query stays queued
self._add_executor_group("group1", 3, num_executors=1)
@@ -324,8 +323,8 @@ class TestExecutorGroups(CustomClusterTestSuite):
# Run query to make sure it times out
result = self.execute_query_expect_failure(self.client, QUERY)
expected_error = "Query aborted:Admission for query exceeded timeout 2000ms in " \
- "pool default-pool. Queued reason: No healthy executor groups " \
- "found for pool default-pool."
+ "pool default-pool. Queued reason: Waiting for executors to " \
+ "start. Only DDL queries can currently run."
assert expected_error in str(result)
assert self.coordinator.service.get_metric_value(
"cluster-membership.executor-groups.total-healthy") == 0