You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by la...@apache.org on 2026/08/08 20:00:58 UTC
(impala) branch master updated: IMPALA-12956: Add profile summary output
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new f755cea7b IMPALA-12956: Add profile summary output
f755cea7b is described below
commit f755cea7b427f6e71741b9bb10c69d6c9b500d65
Author: Aleksandr Efimov <dz...@gmail.com>
AuthorDate: Mon May 25 15:23:57 2026 +0300
IMPALA-12956: Add profile summary output
Add a summary profile format that emits one TSV row per archived profile
with query-level fields useful for batches of profile logs.
The summary output extracts metadata, timing, memory, progress, IO,
spill, query status, and statement fields from archived profile trees.
Missing fields are emitted as N/A instead of failing the whole export
or using synthetic zeroes.
The output uses a shared progress formatter and reports duration from
Start Time and End Time when possible. It falls back to the query
timeline when End Time is absent.
It handles both the default V1 profile layout and the experimental V2
profile layout. V2 scan progress uses aggregated counters, while spilled
bytes are summed from host-level counters under Per Node Profiles.
Refactor shared helpers used by the summary path: move
ParseUtil::ParseMemSpec() to parse-mem-spec.cc, extract
ProgressToString() from ImpalaHttpHandler into progress-util, and split
profile archive encoding/decoding into reusable Python helpers for test
fixture generation.
Add regression coverage for V1 and V2 summary output, query id
filtering, TSV column shape, unlimited text length, failed/cancelled
queries, non-zero spilled bytes, and WebUI thrift-profile downloads.
Assisted-by: gpt-5.6-sol (OpenAI Codex)
Change-Id: I35da958106ffb35b14ef867da50ed23ffda0a45b
Reviewed-on: http://gerrit.cloudera.org:8080/24353
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/impala-http-handler.cc | 12 +-
be/src/service/impala-http-handler.h | 3 -
be/src/util/CMakeLists.txt | 4 +-
be/src/util/impala-profile-tool.cc | 542 ++++++++++++++++++++-
be/src/util/{parse-util.cc => parse-mem-spec.cc} | 66 ---
be/src/util/parse-util.cc | 79 ---
be/src/util/progress-util.cc | 36 ++
be/src/util/progress-util.h | 31 ++
be/src/util/summary-util-test.cc | 10 +-
bin/rat_exclude_files.txt | 4 +
lib/python/impala_py_lib/profiles.py | 35 +-
testdata/impala-profiles/README | 16 +-
.../impala_profile_log_cancelled_query | 1 +
.../impala_profile_log_failed_query | 1 +
...rofile_log_tpcds_compute_stats.expected.summary | 5 +
...ile_log_tpcds_compute_stats_v2.expected.summary | 5 +
tests/observability/test_profile_tool.py | 151 +++++-
tests/webserver/test_web_pages.py | 2 +-
18 files changed, 821 insertions(+), 182 deletions(-)
diff --git a/be/src/service/impala-http-handler.cc b/be/src/service/impala-http-handler.cc
index 66c7c776f..e930c1527 100644
--- a/be/src/service/impala-http-handler.cc
+++ b/be/src/service/impala-http-handler.cc
@@ -48,6 +48,7 @@
#include "util/debug-util.h"
#include "util/logging-support.h"
#include "util/pretty-printer.h"
+#include "util/progress-util.h"
#include "util/redactor.h"
#include "util/summary-util.h"
#include "util/time.h"
@@ -572,17 +573,6 @@ void ImpalaHttpHandler::AddQueryRecordTips(Document* document) {
document->GetAllocator());
}
-std::string ImpalaHttpHandler::ProgressToString(int64_t num_completed, int64_t total) {
- stringstream ss;
- ss << num_completed << " / " << total << " (" << setw(4);
- if (num_completed == 0 || total == 0) {
- ss << "0%)";
- } else {
- ss << (100.0 * num_completed / (1.f * total)) << "%)";
- }
- return ss.str();
-}
-
void ImpalaHttpHandler::QueryStateToJson(const QueryStateRecord& record,
Value* value, Document* document, bool inflight) {
Value user(record.effective_user, document->GetAllocator());
diff --git a/be/src/service/impala-http-handler.h b/be/src/service/impala-http-handler.h
index 6f7c6ffe3..423802c92 100644
--- a/be/src/service/impala-http-handler.h
+++ b/be/src/service/impala-http-handler.h
@@ -239,9 +239,6 @@ class ImpalaHttpHandler {
void QueryMemoryHandler(const Webserver::WebRequest& req,
rapidjson::Document* output);
- /// Utility method to print progress something as n/m(xx%).
- std::string ProgressToString(int64_t num_completed, int64_t total);
-
/// Helper method to render a single QueryStateRecord as a Json object Used by
/// QueryStateHandler().
void QueryStateToJson(const QueryStateRecord& record,
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index dcf64ae9c..7e227b12a 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -45,6 +45,8 @@ set(PROFILE_TOOL_UTIL_SRCS
cpu-info.cc
error-msg.cc
os-info.cc
+ parse-mem-spec.cc
+ progress-util.cc
redactor.cc
runtime-profile-archive.cc
runtime-profile.cc
@@ -368,7 +370,7 @@ ADD_UNIFIED_BE_LSAN_TEST(simple-logger-test "SimpleLoggerTest.*")
ADD_UNIFIED_BE_LSAN_TEST(sql-util-test "SqlUtilTest.*")
ADD_UNIFIED_BE_LSAN_TEST(string-parser-test "StringToInt.*:StringToIntWithBase.*:StringToFloat.*:StringToBool.*:StringToDate.*")
ADD_UNIFIED_BE_LSAN_TEST(string-util-test "TruncateDownTest.*:TruncateUpTest.*:CommaSeparatedContainsTest.*:FindUtf8PosForwardTest.*:FindUtf8PosBackwardTest.*:RandomFindUtf8PosTest.*:StringStreamPopTest.*")
-ADD_UNIFIED_BE_LSAN_TEST(summary-util-test "PrintTableTest.*")
+ADD_UNIFIED_BE_LSAN_TEST(summary-util-test "PrintTableTest.*:ProgressToStringTest.*")
ADD_UNIFIED_BE_LSAN_TEST(symbols-util-test "SymbolsUtil.*")
ADD_UNIFIED_BE_LSAN_TEST(system-state-info-test "SystemStateInfoTest.*")
ADD_UNIFIED_BE_LSAN_TEST(sys-info-test "CpuInfoTest.*:DiskInfoTest.*")
diff --git a/be/src/util/impala-profile-tool.cc b/be/src/util/impala-profile-tool.cc
index fc11ba02c..79c059cd0 100644
--- a/be/src/util/impala-profile-tool.cc
+++ b/be/src/util/impala-profile-tool.cc
@@ -15,18 +15,26 @@
// specific language governing permissions and limitations
// under the License.
+#include <algorithm>
+#include <cctype>
#include <cstdint>
+#include <cstdlib>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <boost/algorithm/string/case_conv.hpp>
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <boost/date_time/posix_time/time_parsers.hpp>
#include <gflags/gflags.h>
#include "common/object-pool.h"
#include "gutil/strings/strip.h"
#include "util/os-info.h"
+#include "util/parse-util.h"
+#include "util/pretty-printer.h"
+#include "util/progress-util.h"
#include "util/runtime-profile.h"
#include "util/string-parser.h"
@@ -45,10 +53,13 @@ static const char* USAGE =
"\n"
"The following options are supported:\n"
"Output options:\n"
- "--profile_format={text,json,prettyjson}: controls\n"
+ "--profile_format={text,json,prettyjson,summary}: controls\n"
" text (default): pretty-print in the standard human readable format\n"
" json: output as JSON with one profile per line. Compatible with jsonlines.org\n"
" prettyjson: output as pretty-printed JSON array with one element per object\n"
+ " summary: output one TSV row per profile with query-level summary fields\n"
+ "--summary_text_length=<integer>: maximum text field length in summary output."
+ " If <= 0, the full text is used\n"
"--profile_verbosity={0,1,2,3,4,minimal,legacy,default,extended,full}: control"
" verbosity of profile output. If not set, picks based on profile version\n"
"\n"
@@ -63,7 +74,8 @@ static const char* USAGE =
" query id metadata.\n";
DEFINE_string(
- profile_format, "text", "Profile format to output: either text, json or prettyjson");
+ profile_format, "text",
+ "Profile format to output: either text, json, prettyjson or summary");
DEFINE_string(profile_verbosity, "", "Verbosity of profile output. Must be one of "
"{0,1,2,3,4,minimal,legacy,default,extended,full}. If not set, picks based on "
"version of each input profile.");
@@ -72,6 +84,8 @@ DEFINE_int64(min_timestamp, -1,
"Minimum timestamp in Unix epoch milliseconds (inclusive) to output profiles for");
DEFINE_int64(max_timestamp, -1,
"Maximum timestamp in Unix epoch milliseconds (inclusive) to output profiles for");
+DEFINE_int32(summary_text_length, 250,
+ "Maximum text field length in summary output. If <= 0, the full text is used.");
using namespace impala;
@@ -82,6 +96,511 @@ using std::cerr;
using std::cin;
using std::cout;
using std::istringstream;
+using std::ostream;
+
+static const char* const SUMMARY_HEADER[] = {"Query ID", "User", "Default Db",
+ "Query Type", "Start Time", "End Time", "Duration", "Queued Duration",
+ "Mem Usage", "Mem Estimate", "Scan Progress", "Query Progress", "Bytes Read",
+ "Bytes Sent", "Bytes Spilled", "State", "Query Status", "# rows fetched",
+ "Resource Pool", "Statement"};
+static const char* const SUMMARY_VALUE_UNAVAILABLE = "N/A";
+
+static bool StartsWith(const string& str, const string& prefix) {
+ return str.compare(0, prefix.size(), prefix) == 0;
+}
+
+static string GetProfileQueryId(const TRuntimeProfileTree& tree) {
+ if (tree.nodes.empty()) return SUMMARY_VALUE_UNAVAILABLE;
+
+ const string& root_name = tree.nodes[0].name;
+ const string prefix = "Query (id=";
+ const string suffix = ")";
+ if (!StartsWith(root_name, prefix)
+ || root_name.size() <= prefix.size() + suffix.size()) {
+ return SUMMARY_VALUE_UNAVAILABLE;
+ }
+ size_t suffix_pos = root_name.size() - suffix.size();
+ if (root_name.compare(suffix_pos, suffix.size(), suffix) != 0) {
+ return SUMMARY_VALUE_UNAVAILABLE;
+ }
+ return root_name.substr(prefix.size(),
+ root_name.size() - prefix.size() - suffix.size());
+}
+
+static const TRuntimeProfileNode* FindNodeByName(
+ const TRuntimeProfileTree& tree, const string& name) {
+ for (const TRuntimeProfileNode& node : tree.nodes) {
+ if (node.name == name) return &node;
+ }
+ return nullptr;
+}
+
+static const TRuntimeProfileNode* FindNodeByPrefix(
+ const TRuntimeProfileTree& tree, const string& prefix) {
+ for (const TRuntimeProfileNode& node : tree.nodes) {
+ if (StartsWith(node.name, prefix)) return &node;
+ }
+ return nullptr;
+}
+
+static string GetInfoString(
+ const TRuntimeProfileNode* node, const string& key,
+ const string& default_value = "") {
+ if (node == nullptr) return default_value;
+ auto it = node->info_strings.find(key);
+ return it == node->info_strings.end() || it->second.empty() ? default_value :
+ it->second;
+}
+
+static bool FindCounterValue(
+ const TRuntimeProfileNode* node, const string& name, int64_t* value) {
+ if (node == nullptr) return false;
+ for (const TCounter& counter : node->counters) {
+ if (counter.name == name) {
+ *value = counter.value;
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool IsAggregatedProfile(const TRuntimeProfileTree& tree) {
+ return tree.__isset.profile_version && tree.profile_version >= 2;
+}
+
+static bool SumAggregatedCounterValues(
+ const TRuntimeProfileTree& tree, const string& name, int64_t* value) {
+ if (!IsAggregatedProfile(tree)) return false;
+ *value = 0;
+ bool found = false;
+ // TRuntimeProfileTree::nodes is a single flat list containing every node in
+ // the profile tree, laid out in pre-order traversal order.
+ for (const TRuntimeProfileNode& node : tree.nodes) {
+ if (node.__isset.aggregated && node.aggregated.__isset.counters) {
+ for (const TAggCounter& counter : node.aggregated.counters) {
+ if (counter.name != name) continue;
+ size_t num_values = std::min(counter.values.size(), counter.has_value.size());
+ for (size_t i = 0; i < num_values; ++i) {
+ if (counter.has_value[i]) {
+ *value += counter.values[i];
+ found = true;
+ }
+ }
+ }
+ }
+ }
+ return found;
+}
+
+static size_t NextSiblingIndex(
+ const vector<TRuntimeProfileNode>& nodes, size_t node_idx) {
+ size_t next_idx = node_idx;
+ int64_t nodes_remaining = 1;
+ while (nodes_remaining > 0 && next_idx < nodes.size()) {
+ nodes_remaining += nodes[next_idx].num_children;
+ ++next_idx;
+ --nodes_remaining;
+ }
+ return next_idx;
+}
+
+static bool SumScratchBytesWrittenForHosts(
+ const vector<TRuntimeProfileNode>& nodes, size_t per_node_idx, int64_t* value) {
+ bool found = false;
+ const TRuntimeProfileNode& per_node = nodes[per_node_idx];
+ size_t host_idx = per_node_idx + 1;
+ for (int child = 0; child < per_node.num_children && host_idx < nodes.size();
+ ++child) {
+ int64_t counter_value = 0;
+ if (FindCounterValue(&nodes[host_idx], "ScratchBytesWritten", &counter_value)) {
+ *value += counter_value;
+ found = true;
+ }
+ host_idx = NextSiblingIndex(nodes, host_idx);
+ }
+ return found;
+}
+
+static bool SumScratchBytesWrittenForPerNodeProfiles(
+ const TRuntimeProfileTree& tree, int64_t* value) {
+ *value = 0;
+ bool found = false;
+ for (size_t i = 0; i < tree.nodes.size(); ++i) {
+ const TRuntimeProfileNode& node = tree.nodes[i];
+ if (!StartsWith(node.name, "Execution Profile ")) continue;
+
+ size_t child_idx = i + 1;
+ for (int child = 0; child < node.num_children && child_idx < tree.nodes.size();
+ ++child) {
+ size_t next_child_idx = NextSiblingIndex(tree.nodes, child_idx);
+ if (tree.nodes[child_idx].name == "Per Node Profiles") {
+ found |= SumScratchBytesWrittenForHosts(tree.nodes, child_idx, value);
+ }
+ child_idx = next_child_idx;
+ }
+ }
+ return found;
+}
+
+static bool SumCounterValuesInSubtreePrefixRecursive(
+ const vector<TRuntimeProfileNode>& nodes, size_t* node_idx, bool in_matching_subtree,
+ const string& prefix, const string& counter_name, int64_t* value) {
+ const TRuntimeProfileNode& node = nodes[*node_idx];
+ bool current_in_matching_subtree = in_matching_subtree || StartsWith(node.name, prefix);
+ bool found = false;
+ if (current_in_matching_subtree) {
+ int64_t counter_value = 0;
+ if (FindCounterValue(&node, counter_name, &counter_value)) {
+ *value += counter_value;
+ found = true;
+ }
+ }
+
+ ++(*node_idx);
+ for (int i = 0; i < node.num_children && *node_idx < nodes.size(); ++i) {
+ found |= SumCounterValuesInSubtreePrefixRecursive(
+ nodes, node_idx, current_in_matching_subtree, prefix, counter_name, value);
+ }
+ return found;
+}
+
+static bool SumCounterValuesInSubtreePrefix(const TRuntimeProfileTree& tree,
+ const string& prefix, const string& counter_name, int64_t* value) {
+ *value = 0;
+ bool found = false;
+ size_t node_idx = 0;
+ while (node_idx < tree.nodes.size()) {
+ found |= SumCounterValuesInSubtreePrefixRecursive(
+ tree.nodes, &node_idx, false, prefix, counter_name, value);
+ }
+ return found;
+}
+
+static bool SumScanRangesComplete(const TRuntimeProfileTree& tree, int64_t* value) {
+ if (IsAggregatedProfile(tree)) {
+ return SumAggregatedCounterValues(tree, "ScanRangesComplete", value);
+ }
+ // V1 profiles include per-instance scan counters alongside averaged counters.
+ return SumCounterValuesInSubtreePrefix(tree, "Instance ", "ScanRangesComplete", value);
+}
+
+static bool SumScratchBytesWritten(const TRuntimeProfileTree& tree, int64_t* value) {
+ return SumScratchBytesWrittenForPerNodeProfiles(tree, value);
+}
+
+static bool GetEventTime(
+ const TEventSequence& sequence, const string& label, int64_t* timestamp) {
+ size_t num_events = std::min(sequence.labels.size(), sequence.timestamps.size());
+ for (size_t i = 0; i < num_events; ++i) {
+ if (sequence.labels[i] == label) {
+ *timestamp = sequence.timestamps[i];
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool GetTimelineDuration(
+ const TRuntimeProfileNode* node, const string& name, int64_t* duration) {
+ if (node == nullptr || !node->__isset.event_sequences) return false;
+ for (const TEventSequence& sequence : node->event_sequences) {
+ if (sequence.name != name || sequence.timestamps.empty()) continue;
+ *duration = *std::max_element(sequence.timestamps.begin(), sequence.timestamps.end());
+ return true;
+ }
+ return false;
+}
+
+static bool ParseTimestamp(const string& timestamp,
+ boost::posix_time::ptime* parsed_timestamp) {
+ if (timestamp.empty()) return false;
+
+ string ts = timestamp;
+ size_t dot = ts.find('.', 19);
+ if (dot != string::npos && ts.size() > dot + 7) ts.resize(dot + 7);
+ try {
+ *parsed_timestamp = boost::posix_time::time_from_string(ts);
+ return true;
+ } catch (...) {
+ return false;
+ }
+}
+
+static string QueryDuration(const TRuntimeProfileNode* summary_node) {
+ boost::posix_time::ptime start_time;
+ boost::posix_time::ptime end_time;
+ if (ParseTimestamp(GetInfoString(summary_node, "Start Time"), &start_time)
+ && ParseTimestamp(GetInfoString(summary_node, "End Time"), &end_time)) {
+ boost::posix_time::time_duration duration = end_time - start_time;
+ if (!duration.is_negative()) {
+ return PrettyPrinter::Print(duration.total_microseconds() * 1000, TUnit::TIME_NS);
+ }
+ }
+
+ int64_t duration = 0;
+ if (!GetTimelineDuration(summary_node, "Query Timeline", &duration)) {
+ return SUMMARY_VALUE_UNAVAILABLE;
+ }
+ return PrettyPrinter::Print(duration, TUnit::TIME_NS);
+}
+
+static string QueuedDuration(const TRuntimeProfileNode* summary_node) {
+ if (summary_node == nullptr || !summary_node->__isset.event_sequences) {
+ return SUMMARY_VALUE_UNAVAILABLE;
+ }
+ for (const TEventSequence& sequence : summary_node->event_sequences) {
+ if (sequence.name != "Query Timeline") continue;
+ int64_t submitted = 0;
+ int64_t admitted = 0;
+ if (GetEventTime(sequence, "Submit for admission", &submitted)
+ && GetEventTime(sequence, "Completed admission", &admitted)
+ && admitted >= submitted) {
+ return PrettyPrinter::Print(admitted - submitted, TUnit::TIME_NS);
+ }
+ }
+ return SUMMARY_VALUE_UNAVAILABLE;
+}
+
+static bool ParseBytesValue(const string& value, int64_t* bytes) {
+ string mem_spec = value;
+ mem_spec.erase(std::remove_if(mem_spec.begin(), mem_spec.end(), [](char c) {
+ return std::isspace(static_cast<unsigned char>(c));
+ }), mem_spec.end());
+ if (mem_spec.empty() || StartsWith(mem_spec, "-")) return false;
+
+ bool is_percent = false;
+ int64_t parsed_bytes = ParseUtil::ParseMemSpec(mem_spec, &is_percent, 0);
+ if (is_percent || parsed_bytes < 0) return false;
+ *bytes = parsed_bytes;
+ return true;
+}
+
+static bool SumFileFormatScanRanges(const string& info_string, int multiplier,
+ int64_t* total_scan_ranges) {
+ bool found = false;
+ size_t pos = 0;
+ while (pos < info_string.size()) {
+ size_t colon = info_string.find(':', pos);
+ if (colon == string::npos) break;
+ const char* count_start = info_string.c_str() + colon + 1;
+ char* count_end = nullptr;
+ int64_t count = std::strtoll(count_start, &count_end, 10);
+ if (count_end != count_start && count > 0) {
+ *total_scan_ranges += count * multiplier;
+ found = true;
+ pos = count_end - info_string.c_str();
+ } else {
+ pos = colon + 1;
+ }
+ }
+ return found;
+}
+
+static bool SumFileFormatScanRangesForParentRecursive(
+ const vector<TRuntimeProfileNode>& nodes, size_t* node_idx, bool parent_matches,
+ const string& parent_name, int64_t* total_scan_ranges) {
+ const TRuntimeProfileNode& node = nodes[*node_idx];
+ bool found = false;
+ if (parent_matches) {
+ string file_formats = GetInfoString(&node, "File Formats");
+ if (!file_formats.empty()) {
+ found |= SumFileFormatScanRanges(file_formats, 1, total_scan_ranges);
+ }
+ }
+
+ bool current_matches = StartsWith(node.name, parent_name);
+ ++(*node_idx);
+ for (int i = 0; i < node.num_children && *node_idx < nodes.size(); ++i) {
+ found |= SumFileFormatScanRangesForParentRecursive(
+ nodes, node_idx, current_matches, parent_name, total_scan_ranges);
+ }
+ return found;
+}
+
+static bool SumFileFormatScanRangesForParent(
+ const TRuntimeProfileTree& tree, const string& parent_name, int64_t* value) {
+ *value = 0;
+ bool found = false;
+ size_t node_idx = 0;
+ while (node_idx < tree.nodes.size()) {
+ found |= SumFileFormatScanRangesForParentRecursive(
+ tree.nodes, &node_idx, false, parent_name, value);
+ }
+ return found;
+}
+
+static bool SumAggregatedFileFormatScanRanges(
+ const TRuntimeProfileTree& tree, int64_t* value) {
+ if (!IsAggregatedProfile(tree)) return false;
+ *value = 0;
+ bool found = false;
+ for (const TRuntimeProfileNode& node : tree.nodes) {
+ if (!node.__isset.aggregated || !node.aggregated.__isset.info_strings) continue;
+ auto file_formats = node.aggregated.info_strings.find("File Formats");
+ if (file_formats == node.aggregated.info_strings.end()) continue;
+ for (const auto& entry : file_formats->second) {
+ found |= SumFileFormatScanRanges(
+ entry.first, static_cast<int>(entry.second.size()), value);
+ }
+ }
+ return found;
+}
+
+static bool SumTotalScanRanges(const TRuntimeProfileTree& tree, int64_t* value) {
+ if (IsAggregatedProfile(tree)) return SumAggregatedFileFormatScanRanges(tree, value);
+ return SumFileFormatScanRangesForParent(tree, "Instance ", value);
+}
+
+static bool ParsePerNodeMemoryUsage(const string& info_string, int64_t* total_bytes) {
+ *total_bytes = 0;
+ bool found = false;
+ size_t pos = 0;
+ while (pos < info_string.size()) {
+ size_t open = info_string.find('(', pos);
+ if (open == string::npos) break;
+ size_t close = info_string.find(')', open + 1);
+ if (close == string::npos) break;
+ int64_t bytes = 0;
+ if (ParseBytesValue(info_string.substr(open + 1, close - open - 1), &bytes)) {
+ found = true;
+ *total_bytes += bytes;
+ }
+ pos = close + 1;
+ }
+ return found;
+}
+
+static string MemoryUsage(const TRuntimeProfileNode* execution_node) {
+ int64_t total_bytes = 0;
+ // Archived profiles keep query-level peak memory as this coordinator info string.
+ if (ParsePerNodeMemoryUsage(
+ GetInfoString(execution_node, "Per Node Peak Memory Usage"), &total_bytes)) {
+ return PrettyPrinter::Print(total_bytes, TUnit::BYTES);
+ }
+ return SUMMARY_VALUE_UNAVAILABLE;
+}
+
+static string MemoryEstimate(const TRuntimeProfileNode* summary_node) {
+ return GetInfoString(
+ summary_node, "Cluster Memory Admitted", SUMMARY_VALUE_UNAVAILABLE);
+}
+
+static string ScanProgress(const TRuntimeProfileTree& tree, const string& query_state) {
+ int64_t completed_scan_ranges = 0;
+ if (!SumScanRangesComplete(tree, &completed_scan_ranges) || completed_scan_ranges < 0) {
+ return SUMMARY_VALUE_UNAVAILABLE;
+ }
+
+ int64_t total_scan_ranges = 0;
+ if (SumTotalScanRanges(tree, &total_scan_ranges) && total_scan_ranges > 0) {
+ return ProgressToString(completed_scan_ranges, total_scan_ranges);
+ }
+
+ // Older profile logs may not keep total scan ranges, but finished queries are 100%
+ // done.
+ if (query_state == "FINISHED" && completed_scan_ranges > 0) {
+ return ProgressToString(completed_scan_ranges, completed_scan_ranges);
+ }
+ return SUMMARY_VALUE_UNAVAILABLE;
+}
+
+static string QueryProgress(
+ const TRuntimeProfileNode* execution_node, const string& query_state) {
+ int64_t num_fragment_instances = 0;
+ if (query_state == "FINISHED"
+ && FindCounterValue(
+ execution_node, "NumFragmentInstances", &num_fragment_instances)
+ && num_fragment_instances > 0) {
+ return ProgressToString(num_fragment_instances, num_fragment_instances);
+ }
+ return SUMMARY_VALUE_UNAVAILABLE;
+}
+
+static string CounterValue(
+ const TRuntimeProfileNode* node, const string& counter_name, TUnit::type unit) {
+ int64_t value = 0;
+ if (!FindCounterValue(node, counter_name, &value)) return SUMMARY_VALUE_UNAVAILABLE;
+ return PrettyPrinter::Print(value, unit);
+}
+
+static string BytesSpilled(const TRuntimeProfileTree& tree) {
+ int64_t value = 0;
+ if (!SumScratchBytesWritten(tree, &value)) return SUMMARY_VALUE_UNAVAILABLE;
+ return PrettyPrinter::Print(value, TUnit::BYTES);
+}
+
+static string RowsFetched(const TRuntimeProfileNode* impala_server_node) {
+ int64_t rows_fetched = 0;
+ if (!FindCounterValue(impala_server_node, "NumRowsFetched", &rows_fetched)) {
+ return SUMMARY_VALUE_UNAVAILABLE;
+ }
+ return std::to_string(rows_fetched);
+}
+
+static string SanitizeTsvValue(string value) {
+ for (char& c : value) {
+ if (c == '\t' || c == '\n' || c == '\r') c = ' ';
+ }
+ return value;
+}
+
+static string TruncateSummaryValue(string value) {
+ if (FLAGS_summary_text_length > 0 && value.size() > FLAGS_summary_text_length) {
+ value = value.substr(0, FLAGS_summary_text_length) + "...";
+ }
+ return value;
+}
+
+static void PrintTsvRow(const vector<string>& values, ostream* out) {
+ for (size_t i = 0; i < values.size(); ++i) {
+ if (i > 0) *out << "\t";
+ *out << SanitizeTsvValue(values[i]);
+ }
+ *out << "\n";
+}
+
+static void PrintSummaryHeader(ostream* out) {
+ size_t num_columns = sizeof(SUMMARY_HEADER) / sizeof(SUMMARY_HEADER[0]);
+ PrintTsvRow(vector<string>(SUMMARY_HEADER, SUMMARY_HEADER + num_columns), out);
+}
+
+static void PrintSummaryRow(
+ const string& query_id, const TRuntimeProfileTree& profile_tree, ostream* out) {
+ const TRuntimeProfileNode* summary_node = FindNodeByName(profile_tree, "Summary");
+ const TRuntimeProfileNode* impala_server_node =
+ FindNodeByName(profile_tree, "ImpalaServer");
+ const TRuntimeProfileNode* execution_node =
+ FindNodeByPrefix(profile_tree, "Execution Profile ");
+ string query_state = GetInfoString(summary_node, "Query State");
+ string output_query_id = GetProfileQueryId(profile_tree);
+ if (output_query_id == SUMMARY_VALUE_UNAVAILABLE && !query_id.empty()) {
+ output_query_id = query_id;
+ }
+
+ vector<string> values = {output_query_id,
+ GetInfoString(summary_node, "User", SUMMARY_VALUE_UNAVAILABLE),
+ GetInfoString(summary_node, "Default Db", SUMMARY_VALUE_UNAVAILABLE),
+ GetInfoString(summary_node, "Query Type", SUMMARY_VALUE_UNAVAILABLE),
+ GetInfoString(summary_node, "Start Time", SUMMARY_VALUE_UNAVAILABLE),
+ GetInfoString(summary_node, "End Time", SUMMARY_VALUE_UNAVAILABLE),
+ QueryDuration(summary_node),
+ QueuedDuration(summary_node),
+ MemoryUsage(execution_node),
+ MemoryEstimate(summary_node),
+ ScanProgress(profile_tree, query_state),
+ QueryProgress(execution_node, query_state),
+ CounterValue(execution_node, "TotalBytesRead", TUnit::BYTES),
+ CounterValue(execution_node, "TotalBytesSent", TUnit::BYTES),
+ BytesSpilled(profile_tree),
+ query_state.empty() ? SUMMARY_VALUE_UNAVAILABLE : query_state,
+ TruncateSummaryValue(GetInfoString(
+ summary_node, "Query Status", SUMMARY_VALUE_UNAVAILABLE)),
+ RowsFetched(impala_server_node),
+ GetInfoString(summary_node, "Request Pool", SUMMARY_VALUE_UNAVAILABLE),
+ TruncateSummaryValue(GetInfoString(
+ summary_node, "Sql Statement", SUMMARY_VALUE_UNAVAILABLE))};
+ PrintTsvRow(values, out);
+}
static bool ParseTimestamp(const string& timestamp_str, int64_t* timestamp) {
StringParser::ParseResult result;
@@ -96,7 +615,7 @@ int main(int argc, char** argv) {
string profile_format = to_lower_copy(FLAGS_profile_format);
if (profile_format != "text" && profile_format != "json"
- && profile_format != "prettyjson") {
+ && profile_format != "prettyjson" && profile_format != "summary") {
cerr << "Invalid --profile_format value: '" << profile_format << "'\n\n"
<< DescribeOneFlag(GetCommandLineFlagInfoOrDie("profile_format"));
return 1;
@@ -114,6 +633,7 @@ int main(int argc, char** argv) {
// Init OsInfo for StopWatch used in MemPool.
// TODO: try using a fake MemPool that invokes malloc/free directly.
OsInfo::Init();
+ if (profile_format == "summary") PrintSummaryHeader(&cout);
if (profile_format == "prettyjson") cout << "[\n";
int errors = 0;
int profiles_emitted = 0;
@@ -152,6 +672,20 @@ int main(int argc, char** argv) {
}
}
+ if (profile_format == "summary") {
+ TRuntimeProfileTree profile_tree;
+ Status status =
+ RuntimeProfile::DeserializeFromArchiveString(encoded_profile, &profile_tree);
+ if (!status.ok()) {
+ cerr << "Error parsing entry " << lineno << ": " << status.GetDetail() << "\n";
+ ++errors;
+ continue;
+ }
+ PrintSummaryRow(query_id, profile_tree, &cout);
+ ++profiles_emitted;
+ continue;
+ }
+
ObjectPool pool;
RuntimeProfile* profile;
Status status = RuntimeProfile::CreateFromArchiveString(
@@ -188,7 +722,7 @@ int main(int argc, char** argv) {
profile->ToJson(verbosity, &json_profile);
RuntimeProfile::JsonProfileToString(json_profile, /*pretty=*/false, &cout);
cout << "\n"; // Each JSON document gets a separate line.
- } else {
+ } else if (profile_format == "prettyjson") {
CHECK_EQ("prettyjson", profile_format);
rapidjson::Document json_profile(rapidjson::kObjectType);
profile->ToJson(verbosity, &json_profile);
diff --git a/be/src/util/parse-util.cc b/be/src/util/parse-mem-spec.cc
similarity index 60%
copy from be/src/util/parse-util.cc
copy to be/src/util/parse-mem-spec.cc
index 241b29657..3e640ad26 100644
--- a/be/src/util/parse-util.cc
+++ b/be/src/util/parse-mem-spec.cc
@@ -17,23 +17,10 @@
#include "util/parse-util.h"
-#include <sstream>
-
-#include <boost/algorithm/string/classification.hpp>
-#include <boost/algorithm/string/split.hpp>
-#include <boost/algorithm/string/trim.hpp>
-
-#include "util/codec.h"
-#include "util/mem-info.h"
#include "util/string-parser.h"
#include "common/names.h"
-using boost::algorithm::is_any_of;
-using boost::algorithm::split;
-using boost::algorithm::token_compress_on;
-using boost::algorithm::trim;
-
namespace impala {
int64_t ParseUtil::ParseMemSpec(const string& mem_spec_str, bool* is_percent,
@@ -114,57 +101,4 @@ int64_t ParseUtil::ParseMemSpec(const string& mem_spec_str, bool* is_percent,
return bytes;
}
-Status ParseUtil::ParseCompressionCodec(
- const string& compression_codec, THdfsCompression::type* type,
- std::optional<int>* level) {
- // Acceptable values are:
- // - zstd, gzip :compression_level
- // - codec
- vector<string> tokens;
- split(tokens, compression_codec, is_any_of(":"), token_compress_on);
- if (tokens.size() > 2) return Status("Invalid compression codec value");
-
- string& codec_name = tokens[0];
- trim(codec_name);
- THdfsCompression::type enum_type;
- RETURN_IF_ERROR(GetThriftEnum(
- codec_name, "compression codec", _THdfsCompression_VALUES_TO_NAMES, &enum_type));
-
- *type = enum_type;
-
- if (tokens.size() == 2) {
- StringParser::ParseResult status;
- string& clevel = tokens[1];
- trim(clevel);
- int compression_level = StringParser::StringToInt<int>(
- clevel.c_str(), static_cast<int>(clevel.size()), &status);
-
- if (status == StringParser::PARSE_SUCCESS) {
- Status res = Codec::ValidateCompressionLevel(enum_type, compression_level);
- if (res.ok()) {
- level->emplace(compression_level);
- }
- return res;
- } else {
- return Status(Substitute("Invalid compression level value - $0"
- ", should be an integer", clevel));
- }
- }
- return Status::OK();
-}
-
-// Return all enum values in a string format, e.g. FOO(1), BAR(2), BAZ(3).
-string GetThriftEnumValues(const map<int, const char*>& enum_values_to_names) {
- bool first = true;
- stringstream ss;
- for (const auto& e : enum_values_to_names) {
- if (!first) {
- ss << ", ";
- } else {
- first = false;
- }
- ss << e.second << "(" << e.first << ")";
- }
- return ss.str();
-}
}
diff --git a/be/src/util/parse-util.cc b/be/src/util/parse-util.cc
index 241b29657..cbb66f0be 100644
--- a/be/src/util/parse-util.cc
+++ b/be/src/util/parse-util.cc
@@ -24,7 +24,6 @@
#include <boost/algorithm/string/trim.hpp>
#include "util/codec.h"
-#include "util/mem-info.h"
#include "util/string-parser.h"
#include "common/names.h"
@@ -36,84 +35,6 @@ using boost::algorithm::trim;
namespace impala {
-int64_t ParseUtil::ParseMemSpec(const string& mem_spec_str, bool* is_percent,
- int64_t relative_reference) {
- *is_percent = false;
- if (mem_spec_str.empty()) return 0;
-
- int64_t multiplier = -1;
- int32_t number_str_len = mem_spec_str.size();
-
- // Look for an accepted suffix such as "MB", "M", or "%".
- string::const_reverse_iterator suffix_char = mem_spec_str.rbegin();
- if (*suffix_char == 'b' || *suffix_char == 'B') {
- // Skip "B", the default is bytes anyways.
- if (suffix_char == mem_spec_str.rend()) return -1;
- suffix_char++;
- number_str_len--;
- }
- switch (*suffix_char) {
- case 't':
- case 'T':
- // Terabytes.
- number_str_len--;
- multiplier = 1024L * 1024L * 1024L * 1024L;
- break;
- case 'g':
- case 'G':
- // Gigabytes.
- number_str_len--;
- multiplier = 1024L * 1024L * 1024L;
- break;
- case 'm':
- case 'M':
- // Megabytes.
- number_str_len--;
- multiplier = 1024L * 1024L;
- break;
- case 'k':
- case 'K':
- // Kilobytes
- number_str_len--;
- multiplier = 1024L;
- break;
- case '%':
- // Don't allow a suffix of "%B".
- if (suffix_char != mem_spec_str.rbegin()) return -1;
- number_str_len--;
- *is_percent = true;
- break;
- // The default is bytes. If there was a trailing "B" it was handled above.
- }
-
- StringParser::ParseResult result;
- int64_t bytes;
- if (multiplier != -1) {
- // Parse float - MB or GB
- double limit_val = StringParser::StringToFloat<double>(mem_spec_str.data(),
- number_str_len, &result);
- if (result != StringParser::PARSE_SUCCESS) return -1;
- bytes = multiplier * limit_val;
- } else {
- // Parse int - bytes or percent
- int64_t limit_val = StringParser::StringToInt<int64_t>(mem_spec_str.data(),
- number_str_len, &result);
- if (result != StringParser::PARSE_SUCCESS) return -1;
-
- if (*is_percent) {
- bytes = (static_cast<double>(limit_val) / 100.0) * relative_reference;
- } else {
- bytes = limit_val;
- }
- }
- // Accept -1 as indicator for infinite memory that we report by a 0 return value.
- if (bytes == -1) {
- return 0;
- }
-
- return bytes;
-}
-
Status ParseUtil::ParseCompressionCodec(
const string& compression_codec, THdfsCompression::type* type,
std::optional<int>* level) {
diff --git a/be/src/util/progress-util.cc b/be/src/util/progress-util.cc
new file mode 100644
index 000000000..b45f39a0e
--- /dev/null
+++ b/be/src/util/progress-util.cc
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/progress-util.h"
+
+#include <iomanip>
+#include <sstream>
+
+#include "common/names.h"
+
+using namespace impala;
+
+string impala::ProgressToString(int64_t num_completed, int64_t total) {
+ stringstream ss;
+ ss << num_completed << " / " << total << " (" << setw(4);
+ if (num_completed == 0 || total == 0) {
+ ss << "0" << "%)";
+ } else {
+ ss << (100.0 * num_completed / static_cast<double>(total)) << "%)";
+ }
+ return ss.str();
+}
diff --git a/be/src/util/progress-util.h b/be/src/util/progress-util.h
new file mode 100644
index 000000000..7916a09ea
--- /dev/null
+++ b/be/src/util/progress-util.h
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef IMPALA_UTIL_PROGRESS_UTIL_H
+#define IMPALA_UTIL_PROGRESS_UTIL_H
+
+#include <cstdint>
+#include <string>
+
+namespace impala {
+
+/// Utility method to print progress as n/m(xx%).
+std::string ProgressToString(int64_t num_completed, int64_t total);
+
+}
+
+#endif
diff --git a/be/src/util/summary-util-test.cc b/be/src/util/summary-util-test.cc
index 2bb0af663..80a3ca90e 100644
--- a/be/src/util/summary-util-test.cc
+++ b/be/src/util/summary-util-test.cc
@@ -18,6 +18,7 @@
#include "testutil/gtest-util.h"
#include "gen-cpp/ExecStats_types.h"
+#include "util/progress-util.h"
#include "util/summary-util.h"
namespace impala {
@@ -110,4 +111,11 @@ TEST(PrintTableTest, HappyPath) {
EXPECT_EQ(expected_str, actual);
}
-} // namespace impala
\ No newline at end of file
+TEST(ProgressToStringTest, BasicProgress) {
+ EXPECT_EQ("0 / 10 ( 0%)", ProgressToString(0, 10));
+ // Keep the percentage width-aligned with the Web UI progress formatter.
+ EXPECT_EQ("5 / 10 ( 50%)", ProgressToString(5, 10));
+ EXPECT_EQ("10 / 0 ( 0%)", ProgressToString(10, 0));
+}
+
+} // namespace impala
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index c2647212a..491c3f1c6 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -187,6 +187,7 @@ testdata/impala-profiles/impala_profile_log_tpcds_compute_stats
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats.expected.txt
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats.expected.json
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats.expected.pretty.json
+testdata/impala-profiles/impala_profile_log_tpcds_compute_stats.expected.summary
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_extended.expected.pretty.json
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_default.expected.txt
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_extended.expected.txt
@@ -197,8 +198,11 @@ testdata/impala-profiles/legacy-profiles/impala_profile_log_tpcds_compute_stats.
testdata/impala-profiles/legacy-profiles/impala_profile_log_tpcds_compute_stats_extended.expected.pretty.json
testdata/impala-profiles/legacy-profiles/impala_profile_log_tpcds_compute_stats_default.expected.txt
testdata/impala-profiles/legacy-profiles/impala_profile_log_tpcds_compute_stats_extended.expected.txt
+testdata/impala-profiles/impala_profile_log_cancelled_query
+testdata/impala-profiles/impala_profile_log_failed_query
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2.expected.json
+testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2.expected.summary
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2_default.expected.txt
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2_extended.expected.txt
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2_extended.expected.pretty.json
diff --git a/lib/python/impala_py_lib/profiles.py b/lib/python/impala_py_lib/profiles.py
index 593c55f02..eabe41e47 100644
--- a/lib/python/impala_py_lib/profiles.py
+++ b/lib/python/impala_py_lib/profiles.py
@@ -22,23 +22,14 @@ import datetime
import zlib
from thrift.protocol import TCompactProtocol
-from thrift.TSerialization import deserialize
+from thrift.TSerialization import deserialize, serialize
from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileTree
-def decode_profile_line(line):
- space_separated = line.split(" ")
- if len(space_separated) == 3:
- ts = int(space_separated[0])
- print(datetime.datetime.fromtimestamp(ts / 1000.0).isoformat(), space_separated[1])
- base64_encoded = space_separated[2]
- elif len(space_separated) == 1:
- base64_encoded = space_separated[0]
- else:
- raise Exception("Unexpected line: " + line)
+def decode_profile_archive(base64_encoded):
possibly_compressed = base64.b64decode(base64_encoded)
- # Handle both compressed and uncompressed Thrift profiles
+ # Handle both compressed and uncompressed Thrift profiles.
try:
thrift = zlib.decompress(possibly_compressed)
except zlib.error:
@@ -47,5 +38,23 @@ def decode_profile_line(line):
tree = TRuntimeProfileTree()
deserialize(tree, thrift, protocol_factory=TCompactProtocol.TCompactProtocolFactory())
tree.validate()
-
return tree
+
+
+def encode_profile_archive(tree):
+ tree.validate()
+ thrift = serialize(tree, protocol_factory=TCompactProtocol.TCompactProtocolFactory())
+ return base64.b64encode(zlib.compress(thrift)).decode('ascii')
+
+
+def decode_profile_line(line):
+ space_separated = line.split(" ")
+ if len(space_separated) == 3:
+ ts = int(space_separated[0])
+ print(datetime.datetime.fromtimestamp(ts / 1000.0).isoformat(), space_separated[1])
+ base64_encoded = space_separated[2]
+ elif len(space_separated) == 1:
+ base64_encoded = space_separated[0]
+ else:
+ raise Exception("Unexpected line: " + line)
+ return decode_profile_archive(base64_encoded)
diff --git a/testdata/impala-profiles/README b/testdata/impala-profiles/README
index c6e707908..f9a0510ea 100644
--- a/testdata/impala-profiles/README
+++ b/testdata/impala-profiles/README
@@ -3,7 +3,7 @@ An impala profile log with the main query and child queries for
compute stats tpcds_parquet.store_sales
This log has associated expected output files (with "expected") in the filename
that were generated by impala-profile-tool with various output formats and verbosity
-levels. These are used in test_profile_tool.py.
+levels, including summary output. These are used in test_profile_tool.py.
impala_profile_log_tpcds_compute_stats_v2:
@@ -12,4 +12,16 @@ compute stats tpcds_parquet.store_sales from an impalad running with
aggregated_profile=true.
This log has associated expected output files (with "expected") in the filename
that were generated by impala-profile-tool with various output formats and verbosity
-levels. These are used in test_profile_tool.py.
+levels, including summary output. These are used in test_profile_tool.py.
+
+
+impala_profile_log_failed_query:
+A minimal synthetic profile log containing one failed query. It is used by
+test_profile_tool.py to cover summary output for missing End Time, Query Status,
+and partial progress values.
+
+
+impala_profile_log_cancelled_query:
+A minimal synthetic profile log containing one cancelled query. It is used by
+test_profile_tool.py to cover summary output for missing End Time, Query Status,
+and partial progress values on cancelled queries.
diff --git a/testdata/impala-profiles/impala_profile_log_cancelled_query b/testdata/impala-profiles/impala_profile_log_cancelled_query
new file mode 100644
index 000000000..5bc5c9d10
--- /dev/null
+++ b/testdata/impala-profiles/impala_profile_log_cancelled_query
@@ -0,0 +1 @@
+1700000001000 0000000000000003:0000000000000004 eJyFUl1v0zAU9dDo2lA6PpYoYUPyY4tYlRZpQpV4YG0nTeqqrmmFEEiTm9x0EY7d2Q7Qtz7wsH/DL4FfwQ/hJplg4wXLD/a5vueee669kfvyPAO1ps0keuPfXZ3eP0C3ZVe8urP1ZJ941X1C3J0gS1Om1jYp4dq1FRimDJ0lKTzv+l3/0O/gpr7fK3b7D1d1KKLiGbFKAbP1Cu6fz4fT9w9KAJkM1Ppvx/3haDQc1P+imf7QZyIEziGiizXNNKgevSriYRlgJpGCKkBQm/wVxFIBZZzTWLFlCsJoGsp0xSEP317bc6SrGcw7zImtAcQs44YOFjtReaxPS146kZI3lJSmnUJ6wZM0MQ+DK15Kz2s0NXAIDVbK8PKihcVlSuNMhLk+xtsoyGDjOvfTO3AbN16gLzwR4B2RzbfvF5ufv37c817vlkGdLbAOqn48 [...]
diff --git a/testdata/impala-profiles/impala_profile_log_failed_query b/testdata/impala-profiles/impala_profile_log_failed_query
new file mode 100644
index 000000000..0a49d9377
--- /dev/null
+++ b/testdata/impala-profiles/impala_profile_log_failed_query
@@ -0,0 +1 @@
+1700000000000 0000000000000001:0000000000000002 eJyFUl1v0zAU9dDo2lA6PpYoYUPyY4tYlRZUoUq8jHVSpa1kTStAIE1ecttaOPbmD7a+9YGH/Rt+CfwKfghOMmDjBcsPvse+555zr4ND//mxAbnETZq+Dm+vTv8foNtyK0HdW3u0jYLqNkL+RmyyjMili0q4duXEmkiNJzSDp92wG+6GHbtxGPaL3f7DVR3wtHiGnFLAZHkGd4+ng/GHeyVgmTTUBu/fDKLJ8O2o/hc16uMRZMJGjGZUY7hMAFJI+zgRhqWYC40JYyKxBLj30hbFR3v4guqFML9fUz7HegH4vGDNbtCtTxXImgald409OfswI4ZpvH+6kZbH+hhsmtI4EoI1pBC6bQlOiuz78TkrpWfAdVMBg0TnumzwrIVnUmR4ZniiqeCEta1MbY2rvJ/Bjt+47oXtC6Mcgh5aff12svrx8/ud4NVmeanM [...]
diff --git a/testdata/impala-profiles/impala_profile_log_tpcds_compute_stats.expected.summary b/testdata/impala-profiles/impala_profile_log_tpcds_compute_stats.expected.summary
new file mode 100644
index 000000000..c14d8695d
--- /dev/null
+++ b/testdata/impala-profiles/impala_profile_log_tpcds_compute_stats.expected.summary
@@ -0,0 +1,5 @@
+Query ID User Default Db Query Type Start Time End Time Duration Queued Duration Mem Usage Mem Estimate Scan Progress Query Progress Bytes Read Bytes Sent Bytes Spilled State Query Status # rows fetched Resource Pool Statement
+5644dcaf9cc34cea:1147591800000000 surya default SET 2026-08-02 18:54:55.097578000 2026-08-02 18:54:55.212170000 114.592ms N/A N/A N/A N/A N/A N/A N/A N/A FINISHED OK 0 N/A set all
+7040d1f43aed2f6d:25a35ee300000000 surya default QUERY 2026-08-02 18:55:00.022451000 2026-08-02 18:55:02.638196000 2s615ms 3.912ms 55.02 MB 455.92 MB 1824 / 1824 ( 100%) 25 / 25 ( 100%) 142.85 MB 52.02 KB 0 FINISHED OK 1824 default-pool SELECT COUNT(*), ss_sold_date_sk FROM tpcds_parquet.store_sales GROUP BY ss_sold_date_sk
+2a41692e6b0100de:a56e272500000000 surya default QUERY 2026-08-02 18:55:06.037338000 2026-08-02 18:55:08.014916000 1s977ms 3.428ms 104.62 MB 192.39 MB 1824 / 1824 ( 100%) 13 / 13 ( 100%) 332.59 MB 204.94 KB 0 FINISHED OK 1 default-pool SELECT NDV(ss_sold_time_sk) AS ss_sold_time_sk, COUNT(CASE WHEN ss_sold_time_sk IS NULL THEN 1 ELSE NULL END), 4, CAST(4 as DOUBLE), NULL, NULL, NDV(ss_item_sk) AS ss_item_sk, COUNT(CASE WHEN ss_item_sk IS NULL THEN 1 ELSE NULL END), 8, CAST(8 as DOU...
+ad4f10adfca635b9:375d926900000000 surya default DDL 2026-08-02 18:55:12.991628000 2026-08-02 18:55:14.641600000 1s649ms N/A N/A N/A N/A N/A N/A N/A N/A FINISHED OK 0 N/A compute stats tpcds_parquet.store_sales
diff --git a/testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2.expected.summary b/testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2.expected.summary
new file mode 100644
index 000000000..2cc276444
--- /dev/null
+++ b/testdata/impala-profiles/impala_profile_log_tpcds_compute_stats_v2.expected.summary
@@ -0,0 +1,5 @@
+Query ID User Default Db Query Type Start Time End Time Duration Queued Duration Mem Usage Mem Estimate Scan Progress Query Progress Bytes Read Bytes Sent Bytes Spilled State Query Status # rows fetched Resource Pool Statement
+84496aa93d193345:60c52ddf00000000 surya default SET 2026-08-02 18:51:44.457757000 2026-08-02 18:51:44.564465000 106.708ms N/A N/A N/A N/A N/A N/A N/A N/A FINISHED OK 0 N/A set all
+564ae7b03a77a9cc:521cf35d00000000 surya default QUERY 2026-08-02 18:51:50.921811000 2026-08-02 18:51:53.702367000 2s780ms 3.858ms 55.01 MB 455.92 MB 1824 / 1824 ( 100%) 25 / 25 ( 100%) 142.85 MB 51.97 KB 0 FINISHED OK 1824 default-pool SELECT COUNT(*), ss_sold_date_sk FROM tpcds_parquet.store_sales GROUP BY ss_sold_date_sk
+614172e3da200bab:f550d5fc00000000 surya default QUERY 2026-08-02 18:52:04.302820000 2026-08-02 18:52:06.386388000 2s083ms 3.369ms 104.63 MB 192.39 MB 1824 / 1824 ( 100%) 13 / 13 ( 100%) 332.59 MB 205.12 KB 0 FINISHED OK 1 default-pool SELECT NDV(ss_sold_time_sk) AS ss_sold_time_sk, COUNT(CASE WHEN ss_sold_time_sk IS NULL THEN 1 ELSE NULL END), 4, CAST(4 as DOUBLE), NULL, NULL, NDV(ss_item_sk) AS ss_item_sk, COUNT(CASE WHEN ss_item_sk IS NULL THEN 1 ELSE NULL END), 8, CAST(8 as DOU...
+424b230a7f178a7f:ed8da05500000000 surya default DDL 2026-08-02 18:52:12.782567000 2026-08-02 18:52:14.341230000 1s558ms N/A N/A N/A N/A N/A N/A N/A N/A FINISHED OK 0 N/A compute stats tpcds_parquet.store_sales
diff --git a/tests/observability/test_profile_tool.py b/tests/observability/test_profile_tool.py
index d37c08ce4..6459ff2a3 100644
--- a/tests/observability/test_profile_tool.py
+++ b/tests/observability/test_profile_tool.py
@@ -17,12 +17,14 @@
import os.path
import tempfile
-from subprocess import PIPE, Popen, check_call
+from subprocess import PIPE, Popen, check_call, check_output
+from impala_py_lib import profiles
from tests.common.environ import impalad_basedir
from tests.common.base_test_suite import BaseTestSuite
IMPALA_HOME = os.environ['IMPALA_HOME']
+SUMMARY_NUM_COLUMNS = 20
def get_profile_path(filename):
@@ -125,6 +127,12 @@ class TestProfileTool(BaseTestSuite):
self._compare_webui_thrift_profile_output(['--profile_format=prettyjson'],
get_profile_path('impala_profile_log_tpcds_compute_stats'))
+ def test_webui_thrift_profile_summary_output(self):
+ self._compare_webui_thrift_profile_output(['--profile_format=summary'],
+ get_profile_path('impala_profile_log_tpcds_compute_stats'))
+ self._compare_webui_thrift_profile_output(['--profile_format=summary'],
+ get_profile_path('impala_profile_log_tpcds_compute_stats_v2'))
+
def test_webui_thrift_profile_ignores_surrounding_whitespace(self):
self._compare_webui_thrift_profile_output([],
get_profile_path('impala_profile_log_tpcds_compute_stats'),
@@ -183,6 +191,97 @@ class TestProfileTool(BaseTestSuite):
assert 'timestamp prefixes are parsed only when' in stderr
assert '--min_timestamp/--max_timestamp filtering is enabled' in stderr
+ def test_summary_output(self):
+ self._compare_profile_tool_output(['--profile_format=summary'],
+ get_profile_path('impala_profile_log_tpcds_compute_stats'),
+ get_profile_path('impala_profile_log_tpcds_compute_stats.expected.summary'))
+ self._assert_summary_tsv_shape(self._run_profile_tool(['--profile_format=summary'],
+ get_profile_path('impala_profile_log_tpcds_compute_stats')))
+ self._compare_profile_tool_output(['--profile_format=summary'],
+ get_profile_path('impala_profile_log_tpcds_compute_stats_v2'),
+ get_profile_path('impala_profile_log_tpcds_compute_stats_v2.expected.summary'))
+ self._assert_summary_tsv_shape(self._run_profile_tool(['--profile_format=summary'],
+ get_profile_path('impala_profile_log_tpcds_compute_stats_v2')))
+
+ def test_summary_output_with_query_id_filter(self):
+ query_id = '564ae7b03a77a9cc:521cf35d00000000'
+ output = self._run_profile_tool(
+ ['--profile_format=summary', '--query_id=%s' % query_id],
+ get_profile_path('impala_profile_log_tpcds_compute_stats_v2'))
+
+ lines = self._summary_tsv_lines(output)
+ assert len(lines) == 2
+ assert lines[1].split('\t')[0] == query_id
+
+ def test_summary_output_nonzero_bytes_spilled(self):
+ for profile_log in ['impala_profile_log_tpcds_compute_stats',
+ 'impala_profile_log_tpcds_compute_stats_v2']:
+ with tempfile.NamedTemporaryFile() as input_log:
+ self._write_profile_with_scratch_bytes_written(
+ get_profile_path(profile_log), 64 * 1024, input_log)
+ output = self._run_profile_tool(['--profile_format=summary'], input_log.name)
+
+ lines = self._summary_tsv_lines(output)
+ assert len(lines) == 2
+ fields = lines[1].split('\t')
+ assert fields[14] == '64.00 KB'
+
+ def test_summary_output_failed_query(self):
+ output = self._run_profile_tool(
+ ['--profile_format=summary', '--summary_text_length=60'],
+ get_profile_path('impala_profile_log_failed_query'))
+
+ lines = self._summary_tsv_lines(output)
+ assert len(lines) == 2
+ fields = lines[1].split('\t')
+ assert fields[5] == 'N/A'
+ assert fields[6] == '350.000ms'
+ assert fields[10] == 'N/A'
+ assert fields[11] == 'N/A'
+ assert fields[15] == 'EXCEPTION'
+ assert fields[16] == (
+ 'Memory limit exceeded: could not allocate 64.00 MB without e...')
+
+ def test_summary_output_cancelled_query(self):
+ output = self._run_profile_tool(
+ ['--profile_format=summary', '--summary_text_length=60'],
+ get_profile_path('impala_profile_log_cancelled_query'))
+
+ lines = self._summary_tsv_lines(output)
+ assert len(lines) == 2
+ fields = lines[1].split('\t')
+ assert fields[5] == 'N/A'
+ assert fields[6] == '350.000ms'
+ assert fields[10] == 'N/A'
+ assert fields[11] == 'N/A'
+ assert fields[15] == 'CANCELLED'
+ assert fields[16] == (
+ 'Cancelled by user: query cancellation requested before all f...')
+
+ def test_summary_text_length_unlimited(self):
+ output = self._run_profile_tool(
+ ['--profile_format=summary', '--summary_text_length=0'],
+ get_profile_path('impala_profile_log_tpcds_compute_stats_v2'))
+
+ lines = self._summary_tsv_lines(output)
+ assert len(lines) == 5
+ fields = lines[3].split('\t')
+ long_stmt = fields[-1]
+ assert len(long_stmt) > 250
+ assert not long_stmt.endswith('...')
+
+ def _summary_tsv_lines(self, output):
+ if not isinstance(output, str):
+ output = output.decode('utf-8')
+ self._assert_summary_tsv_shape(output)
+ return output.splitlines()
+
+ def _assert_summary_tsv_shape(self, output):
+ if not isinstance(output, str):
+ output = output.decode('utf-8')
+ for line in output.splitlines():
+ assert len(line.split('\t')) == SUMMARY_NUM_COLUMNS
+
def _compare_profile_tool_output(self, args, input_log, expected_output):
"""Run impala-profile-tool on input_log and compare it to the contents of the
file at 'expected_output'."""
@@ -210,10 +309,60 @@ class TestProfileTool(BaseTestSuite):
_timestamp, query_id, encoded_profile = f.readline().split(None, 2)
return query_id, encoded_profile.rstrip()
+ def _write_profile_with_scratch_bytes_written(
+ self, input_log, scratch_bytes_written, output):
+ with open(input_log, 'r') as f:
+ for line in f:
+ timestamp, query_id, encoded_profile = line.split(None, 2)
+ profile_tree = profiles.decode_profile_archive(encoded_profile.rstrip())
+ if self._set_first_per_node_counter(
+ profile_tree, 'ScratchBytesWritten', scratch_bytes_written):
+ encoded_profile = profiles.encode_profile_archive(profile_tree)
+ output.write(('%s %s %s\n' % (
+ timestamp, query_id, encoded_profile)).encode('utf-8'))
+ output.flush()
+ return
+ assert False, 'No Per Node Profiles ScratchBytesWritten counter found'
+
+ def _set_first_per_node_counter(self, profile_tree, counter_name, value):
+ nodes = profile_tree.nodes
+ for idx, node in enumerate(nodes):
+ if not node.name.startswith('Execution Profile '):
+ continue
+
+ child_idx = idx + 1
+ for _ in range(node.num_children):
+ if child_idx >= len(nodes):
+ break
+ next_child_idx = self._next_sibling_index(nodes, child_idx)
+ if nodes[child_idx].name == 'Per Node Profiles':
+ host_idx = child_idx + 1
+ for _ in range(nodes[child_idx].num_children):
+ if host_idx >= len(nodes):
+ break
+ for counter in nodes[host_idx].counters:
+ if counter.name == counter_name:
+ counter.value = value
+ return True
+ host_idx = self._next_sibling_index(nodes, host_idx)
+ child_idx = next_child_idx
+ return False
+
+ def _next_sibling_index(self, nodes, node_idx):
+ next_idx = node_idx
+ nodes_remaining = 1
+ while nodes_remaining > 0 and next_idx < len(nodes):
+ nodes_remaining += nodes[next_idx].num_children
+ next_idx += 1
+ nodes_remaining -= 1
+ return next_idx
+
def _run_profile_tool(self, args, input_log, output=None):
with open(input_log, 'r') as f:
command = [os.path.join(IMPALA_HOME, "bin/run-binary.sh"),
os.path.join(impalad_basedir, 'util/impala-profile-tool')] + args
+ if output is None:
+ return check_output(command, stdin=f)
check_call(command, stdin=f, stdout=output)
output.flush()
diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py
index 0725717c8..d353c4418 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -1494,7 +1494,7 @@ class TestWebPage(ImpalaTestSuite):
assert json_part["state"] == "RUNNING"
assert json_part["waiting"] is False
assert json_part["executing"] is True
- assert json_part["query_progress"] == "0 / 4 ( 0%)"
+ assert json_part["query_progress"] == "0 / 4 ( 0%)"
assert found, "Query {} not found in response_json\n{}".format(
query_id, json.dumps(response_json, sort_keys=True, indent=4))