You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2019/01/07 18:04:22 UTC
[2/2] impala git commit: IMPALA-6742: Profiles of running queries
should include execution summary.
IMPALA-6742: Profiles of running queries should include execution summary.
Currently execution summary is not included in the profiles of running
queries, and it's only reported when the query is finished. This jira makes
the execution summary to the profile reported when queries are still running.
Testing:
Added unit test.
Done with real cluster.
Change-Id: Idc7f714c9427d4b26d4e78cf27ceca2b0b336699
Reviewed-on: http://gerrit.cloudera.org:8080/11591
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e9652a48
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e9652a48
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e9652a48
Branch: refs/heads/master
Commit: e9652a48dd00c3c076ddccaa940d074b6970b7fc
Parents: a91b24c
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Mon Oct 29 21:54:15 2018 -0700
Committer: Joe McDonnell <jo...@cloudera.com>
Committed: Mon Jan 7 17:55:48 2019 +0000
----------------------------------------------------------------------
be/src/service/impala-server.cc | 23 ++++++++++++++++-------
be/src/service/impala-server.h | 3 +++
tests/query_test/test_observability.py | 23 +++++++++++++++++++++++
3 files changed, 42 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/e9652a48/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index c7e52ae..86e40d3 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -664,6 +664,9 @@ Status ImpalaServer::GetRuntimeProfileOutput(const TUniqueId& query_id,
lock_guard<mutex> l(*request_state->lock());
RETURN_IF_ERROR(CheckProfileAccess(user, request_state->effective_user(),
request_state->user_has_profile_access()));
+ if (request_state->GetCoordinator() != nullptr) {
+ UpdateExecSummary(request_state);
+ }
if (format == TRuntimeProfileFormat::BASE64) {
RETURN_IF_ERROR(request_state->profile()->SerializeToArchiveString(output));
} else if (format == TRuntimeProfileFormat::THRIFT) {
@@ -1110,6 +1113,18 @@ Status ImpalaServer::SetQueryInflight(shared_ptr<SessionState> session_state,
return Status::OK();
}
+void ImpalaServer::UpdateExecSummary(
+ std::shared_ptr<ClientRequestState> request_state) const {
+ DCHECK(request_state->GetCoordinator() != nullptr);
+ TExecSummary t_exec_summary;
+ request_state->GetCoordinator()->GetTExecSummary(&t_exec_summary);
+ request_state->summary_profile()->SetTExecSummary(t_exec_summary);
+ string exec_summary = PrintExecSummary(t_exec_summary);
+ request_state->summary_profile()->AddInfoStringRedacted("ExecSummary", exec_summary);
+ request_state->summary_profile()->AddInfoStringRedacted("Errors",
+ request_state->GetCoordinator()->GetErrorLog());
+}
+
Status ImpalaServer::UnregisterQuery(const TUniqueId& query_id, bool check_inflight,
const Status* cause) {
VLOG_QUERY << "UnregisterQuery(): query_id=" << PrintId(query_id);
@@ -1154,13 +1169,7 @@ Status ImpalaServer::UnregisterQuery(const TUniqueId& query_id, bool check_infli
}
if (request_state->GetCoordinator() != nullptr) {
- TExecSummary t_exec_summary;
- request_state->GetCoordinator()->GetTExecSummary(&t_exec_summary);
- request_state->summary_profile()->SetTExecSummary(t_exec_summary);
- string exec_summary = PrintExecSummary(t_exec_summary);
- request_state->summary_profile()->AddInfoStringRedacted("ExecSummary", exec_summary);
- request_state->summary_profile()->AddInfoStringRedacted("Errors",
- request_state->GetCoordinator()->GetErrorLog());
+ UpdateExecSummary(request_state);
}
if (request_state->schedule() != nullptr) {
http://git-wip-us.apache.org/repos/asf/impala/blob/e9652a48/be/src/service/impala-server.h
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index d9b4b05..981df74 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -637,6 +637,9 @@ class ImpalaServer : public ImpalaServiceIf,
Status GetExecSummary(const TUniqueId& query_id, const std::string& user,
TExecSummary* result) WARN_UNUSED_RESULT;
+ /// Collect ExecSummary and update it to the profile in request_state
+ void UpdateExecSummary(std::shared_ptr<ClientRequestState> request_state) const;
+
/// Initialize "default_configs_" to show the default values for ImpalaQueryOptions and
/// "support_start_over/false" to indicate that Impala does not support start over
/// in the fetch call.
http://git-wip-us.apache.org/repos/asf/impala/blob/e9652a48/tests/query_test/test_observability.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index 7f4b2c8..fd44b80 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -216,6 +216,29 @@ class TestObservability(ImpalaTestSuite):
# fetch an exec_summary.
assert exec_summary is not None and exec_summary.nodes is not None
+ def test_exec_summary_in_runtime_profile(self):
+ """Test that the exec summary is populated in runtime profile correctly in every
+ query state"""
+ query = "select count(*) from functional.alltypes"
+ handle = self.execute_query_async(query,
+ {"debug_action": "CRS_BEFORE_ADMISSION:SLEEP@1000"})
+
+ # If ExecuteStatement() has completed and the query is paused in the admission control
+ # phase, then the coordinator has not started yet and exec_summary should be empty.
+ profile = self.client.get_runtime_profile(handle)
+ assert "ExecSummary:" not in profile, profile
+ # After completion of the admission control phase, the coordinator would have started
+ # and we should get a populated exec_summary.
+ self.client.wait_for_admission_control(handle)
+ profile = self.client.get_runtime_profile(handle)
+ assert "ExecSummary:" in profile, profile
+
+ self.client.fetch(query, handle)
+ # After fetching the results and reaching finished state, we should still be able to
+ # fetch an exec_summary in profile.
+ profile = self.client.get_runtime_profile(handle)
+ assert "ExecSummary:" in profile, profile
+
@SkipIfLocal.multiple_impalad
def test_profile_fragment_instances(self):
"""IMPALA-6081: Test that the expected number of fragment instances and their exec