You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ar...@apache.org on 2019/04/11 18:25:18 UTC
[impala] 06/06: IMPALA-7981: Add host disk statistics to profile
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 10e052a681863a08d895c5447c6c7bf8b31156c1
Author: Lars Volker <lv...@cloudera.com>
AuthorDate: Thu Mar 21 08:44:34 2019 -0700
IMPALA-7981: Add host disk statistics to profile
This change adds host disk statistics to profiles. For each host that
participates in the query execution it adds the read and write bandwidth
across all disks. This includes all data read or written by the host as
part of the execution of a query (spilling), by the HDFS data node, and
by other processes running on the same system.
The change adds tests for the added functionality to the backend and end
to end tests.
Change-Id: I373e7da47a0d722938e6ca1572c49a502951ed57
Reviewed-on: http://gerrit.cloudera.org:8080/12822
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/runtime/query-state.cc | 9 ++
be/src/util/disk-info.h | 6 ++
be/src/util/system-state-info-test.cc | 51 ++++++++++
be/src/util/system-state-info.cc | 175 ++++++++++++++++++++++-----------
be/src/util/system-state-info.h | 82 ++++++++++++++-
tests/query_test/test_observability.py | 7 +-
6 files changed, 266 insertions(+), 64 deletions(-)
diff --git a/be/src/runtime/query-state.cc b/be/src/runtime/query-state.cc
index 0d33a12..134b35e 100644
--- a/be/src/runtime/query-state.cc
+++ b/be/src/runtime/query-state.cc
@@ -170,6 +170,15 @@ Status QueryState::Init(const TExecQueryFInstancesParams& exec_rpc_params) {
"HostNetworkTx", TUnit::BYTES_PER_SECOND, [system_state_info] () {
return system_state_info->GetNetworkUsage().tx_rate;
});
+ // Add disk stats
+ host_profile_->AddChunkedTimeSeriesCounter(
+ "HostDiskReadThroughput", TUnit::BYTES_PER_SECOND, [system_state_info] () {
+ return system_state_info->GetDiskStats().read_rate;
+ });
+ host_profile_->AddChunkedTimeSeriesCounter(
+ "HostDiskWriteThroughput", TUnit::BYTES_PER_SECOND, [system_state_info] () {
+ return system_state_info->GetDiskStats().write_rate;
+ });
}
// Starting a new query creates threads and consumes a non-trivial amount of memory.
diff --git a/be/src/util/disk-info.h b/be/src/util/disk-info.h
index cb43b15..42c188e 100644
--- a/be/src/util/disk-info.h
+++ b/be/src/util/disk-info.h
@@ -64,6 +64,12 @@ class DiskInfo {
return disks_[disk_id].is_rotational;
}
+ /// Returns true if the disk with name 'device_name' is known to the DiskInfo class,
+ /// i.e. we consider it to be useful to include in metrics and such.
+ static bool is_known_disk(const std::string& device_name) {
+ return disk_name_to_disk_id_.find(device_name) != disk_name_to_disk_id_.end();
+ }
+
static std::string DebugString();
private:
diff --git a/be/src/util/system-state-info-test.cc b/be/src/util/system-state-info-test.cc
index c8680ba..a02a140 100644
--- a/be/src/util/system-state-info-test.cc
+++ b/be/src/util/system-state-info-test.cc
@@ -17,6 +17,7 @@
#include "common/atomic.h"
#include "testutil/gtest-util.h"
+#include "util/disk-info.h"
#include "util/system-state-info.h"
#include "util/time.h"
@@ -57,6 +58,16 @@ TEST_F(SystemStateInfoTest, ReadProcNetDev) {
EXPECT_GT(state[SystemStateInfo::NET_TX_PACKETS], 0);
}
+// Smoke test to make sure that we read non-zero values from /proc/diskstats.
+TEST_F(SystemStateInfoTest, ReadProcDiskStats) {
+ info.ReadCurrentProcDiskStats();
+ const SystemStateInfo::DiskValues& state = info.disk_values_[info.disk_val_idx_];
+ EXPECT_GT(state[SystemStateInfo::DISK_READS_COMPLETED], 0);
+ EXPECT_GT(state[SystemStateInfo::DISK_SECTORS_READ], 0);
+ EXPECT_GT(state[SystemStateInfo::DISK_WRITES_COMPLETED], 0);
+ EXPECT_GT(state[SystemStateInfo::DISK_SECTORS_WRITTEN], 0);
+}
+
// Tests parsing a line similar to the first line of /proc/stat.
TEST_F(SystemStateInfoTest, ParseProcStat) {
// Fields are: user nice system idle iowait irq softirq steal guest guest_nice
@@ -105,6 +116,24 @@ TEST_F(SystemStateInfoTest, ParseProcNetDevStringCentos6) {
EXPECT_EQ(state[SystemStateInfo::NET_TX_PACKETS], 84770);
}
+// Tests parsing a string similar to the contents of /proc/diskstats. This test also makes
+// sure that values of a partition are not accounted towards the disk it is on.
+TEST_F(SystemStateInfoTest, ParseProcDiskStatsString) {
+ // We cannot hard-code the device name since our Jenkins workers might see different
+ // device names in their virtual environments.
+ string disk_stats =
+ R"( 8 0 $0 17124835 222797 716029974 8414920 43758807 38432095 7867287712 630179264 0 32547524 638999340
+ 8 1 $01 17124482 222797 716027002 8414892 43546943 38432095 7867287712 629089180 0 31590972 637917344)";
+ string device = DiskInfo::device_name(0);
+ cout << Substitute(disk_stats, device);
+ info.ReadProcDiskStatsString(Substitute(disk_stats, device));
+ const SystemStateInfo::DiskValues& state = info.disk_values_[info.disk_val_idx_];
+ EXPECT_EQ(state[SystemStateInfo::DISK_SECTORS_READ], 716029974);
+ EXPECT_EQ(state[SystemStateInfo::DISK_READS_COMPLETED], 17124835);
+ EXPECT_EQ(state[SystemStateInfo::DISK_SECTORS_WRITTEN], 7867287712);
+ EXPECT_EQ(state[SystemStateInfo::DISK_WRITES_COMPLETED], 43758807);
+}
+
// Tests that computing CPU ratios doesn't overflow
TEST_F(SystemStateInfoTest, ComputeCpuRatiosIntOverflow) {
// Load old and new values for CPU counters. These values are from a system where we
@@ -167,5 +196,27 @@ TEST_F(SystemStateInfoTest, ComputeNetworkUsage) {
EXPECT_EQ(n.tx_rate, 12000);
}
+// Tests the computation logic for disk statistics.
+TEST_F(SystemStateInfoTest, ComputeDiskStats) {
+ // Two sets of values in the format of /proc/diskstats. We cannot hard-code the device
+ // name since our Jenkins workers might see different device names in their virtual
+ // environments.
+ string disk_stats_1 = R"( 1 15 ram15 0 0 0 0 0 0 0 0 0 0 0
+ 7 0 loop0 0 0 0 0 0 0 0 0 0 0 0
+ 8 0 $0 0 0 1000 0 0 0 2000 0 0 0 0
+ 8 1 $01 0 0 1000 0 0 0 2000 0 0 0 0)";
+ string disk_stats_2 = R"( 8 0 $0 0 0 2000 0 0 0 4000 0 0 0 0
+ 8 1 $01 0 0 2000 0 0 0 4000 0 0 0 0)";
+
+ string device = DiskInfo::device_name(0);
+ info.ReadProcDiskStatsString(Substitute(disk_stats_1, device));
+ info.ReadProcDiskStatsString(Substitute(disk_stats_2, device));
+ int period_ms = 500;
+ info.ComputeDiskStats(period_ms);
+ const SystemStateInfo::DiskStats& ds = info.GetDiskStats();
+ EXPECT_EQ(ds.read_rate, 2 * 1000 * SystemStateInfo::BYTES_PER_SECTOR);
+ EXPECT_EQ(ds.write_rate, 2 * 2000 * SystemStateInfo::BYTES_PER_SECTOR);
+}
+
} // namespace impala
diff --git a/be/src/util/system-state-info.cc b/be/src/util/system-state-info.cc
index 1194bf4..e808724 100644
--- a/be/src/util/system-state-info.cc
+++ b/be/src/util/system-state-info.cc
@@ -21,12 +21,13 @@
#include "kudu/util/env.h"
#include "kudu/util/faststring.h"
#include "kudu/util/logging.h"
+#include "util/disk-info.h"
#include "util/error-util.h"
#include "util/string-parser.h"
#include "util/system-state-info.h"
#include "util/time.h"
-#include <algorithm>
+#include <numeric>
#include <fstream>
#include <iostream>
@@ -39,43 +40,79 @@ using kudu::ReadFileToString;
using strings::Split;
using strings::SkipWhitespace;
+namespace {
+/// Reads a file into a buffer and returns whether the read was successful. If not
+/// successful, writes a warning into the log.
+bool ReadFileOrWarn(const string& path, faststring* buf) {
+ kudu::Status status = ReadFileToString(Env::Default(), path, buf);
+ if (!status.ok()) {
+ KLOG_EVERY_N_SECS(WARNING, 60) << "Could not open " << path << ": "
+ << status.message() << endl;
+ return false;
+ }
+ return true;
+}
+
+template <class T>
+void ReadIntArray(const StringPiece& sp, int expected_num_values, T* out) {
+ vector<StringPiece> counters = Split(sp, " ", SkipWhitespace());
+
+ // Something is wrong with the number of values that we read
+ if (expected_num_values > counters.size()) {
+ memset(out, 0, sizeof(*out));
+ KLOG_EVERY_N_SECS(WARNING, 60) << "Failed to parse enough values: expected "
+ << expected_num_values << " but found " << counters.size() << ". Input was: "
+ << sp;
+ return;
+ }
+
+ for (int i = 0; i < expected_num_values; ++i) {
+ int64_t* v = &(*out)[i];
+ if (!safe_strto64(counters[i].as_string(), v)) {
+ KLOG_EVERY_N_SECS(WARNING, 60) << "Failed to parse value: " << *v;
+ *v = 0;
+ }
+ DCHECK_GE(*v, 0) << "Value " << i << ": " << *v;
+ }
+}
+} // anonymous namespace
+
namespace impala {
// Partially initializing cpu_ratios_ will default-initialize the remaining members.
SystemStateInfo::SystemStateInfo() {
memset(&cpu_ratios_, 0, sizeof(cpu_ratios_));
memset(&network_usage_, 0, sizeof(network_usage_));
+ memset(&disk_stats_, 0, sizeof(disk_stats_));
ReadCurrentProcStat();
ReadCurrentProcNetDev();
last_net_update_ms_ = MonotonicMillis();
+ ReadCurrentProcDiskStats();
+ last_disk_update_ms_ = MonotonicMillis();
}
void SystemStateInfo::CaptureSystemStateSnapshot() {
// Capture and compute CPU usage
ReadCurrentProcStat();
ComputeCpuRatios();
+
int64_t now = MonotonicMillis();
ReadCurrentProcNetDev();
DCHECK_GT(last_net_update_ms_, 0L);
ComputeNetworkUsage(now - last_net_update_ms_);
last_net_update_ms_ = now;
-}
-int64_t SystemStateInfo::ParseInt64(const string& val) const {
- StringParser::ParseResult result;
- int64_t parsed = StringParser::StringToInt<int64_t>(val.c_str(), val.size(), &result);
- if (result == StringParser::PARSE_SUCCESS) return parsed;
- return -1;
+ now = MonotonicMillis();
+ ReadCurrentProcDiskStats();
+ DCHECK_GT(last_disk_update_ms_, 0L);
+ ComputeDiskStats(now - last_disk_update_ms_);
+ last_disk_update_ms_ = now;
}
void SystemStateInfo::ReadCurrentProcStat() {
string path = "/proc/stat";
faststring buf;
- kudu::Status status = ReadFileToString(Env::Default(), path, &buf);
- if (!status.ok()) {
- LOG(WARNING) << "Could not open " << path << ": " << status.message() << endl;
- return;
- }
+ if (!ReadFileOrWarn(path, &buf)) return;
StringPiece buf_sp(reinterpret_cast<const char*>(buf.data()), buf.size());
vector<StringPiece> lines = strings::Split(buf_sp, "\n");
DCHECK_GT(lines.size(), 0);
@@ -83,21 +120,14 @@ void SystemStateInfo::ReadCurrentProcStat() {
}
void SystemStateInfo::ReadProcStatString(const string& stat_string) {
- stringstream ss(stat_string);
-
- // Skip the first value ('cpu ');
- ss.ignore(5);
-
cpu_val_idx_ = 1 - cpu_val_idx_;
+ DCHECK(cpu_val_idx_ == 0 || cpu_val_idx_ == 1) << "cpu_val_idx_: " << cpu_val_idx_;
CpuValues& next_values = cpu_values_[cpu_val_idx_];
- for (int i = 0; i < NUM_CPU_VALUES; ++i) {
- int64_t v = -1;
- ss >> v;
- DCHECK_GE(v, 0) << "Value " << i << ": " << v;
- // Clamp invalid entries at 0
- next_values[i] = max(v, 0L);
- }
+ // Skip the first value ('cpu ');
+ StringPiece sp(stat_string);
+ DCHECK(sp.starts_with("cpu "));
+ ReadIntArray(sp.substr(5), NUM_CPU_VALUES, &next_values);
}
void SystemStateInfo::ComputeCpuRatios() {
@@ -126,27 +156,19 @@ void SystemStateInfo::ComputeCpuRatios() {
void SystemStateInfo::ReadCurrentProcNetDev() {
string path = "/proc/net/dev";
faststring buf;
- kudu::Status status = ReadFileToString(Env::Default(), path, &buf);
- if (!status.ok()) {
- LOG(WARNING) << "Could not open " << path << ": " << status.message() << endl;
- return;
- }
+ if (!ReadFileOrWarn(path, &buf)) return;
ReadProcNetDevString(buf.ToString());
}
void SystemStateInfo::ReadProcNetDevString(const string& dev_string) {
- stringstream ss(dev_string);
- string line;
-
- // Discard the first two lines that contain the header
- getline(ss, line);
- getline(ss, line);
-
net_val_idx_ = 1 - net_val_idx_;
+ DCHECK(net_val_idx_ == 0 || net_val_idx_ == 1) << "net_val_idx_: " << net_val_idx_;
NetworkValues& sum_values = network_values_[net_val_idx_];
memset(&sum_values, 0, sizeof(sum_values));
- while (getline(ss, line)) {
+ StringPiece sp(dev_string);
+ vector<StringPiece> lines = Split(sp, "\n");
+ for (const StringPiece& line : lines) {
NetworkValues line_values;
ReadProcNetDevLine(line, &line_values);
AddNetworkValues(line_values, &sum_values);
@@ -154,8 +176,8 @@ void SystemStateInfo::ReadProcNetDevString(const string& dev_string) {
}
void SystemStateInfo::ReadProcNetDevLine(
- const string& dev_string, NetworkValues* result) {
- StringPiece sp = StringPiece(dev_string);
+ const StringPiece& dev_string, NetworkValues* result) {
+ StringPiece sp(dev_string);
// Lines can have leading whitespace
StripWhiteSpace(&sp);
@@ -172,23 +194,7 @@ void SystemStateInfo::ReadProcNetDevLine(
return;
}
- vector<StringPiece> counters = Split(sp.substr(colon_idx + 1), " ", SkipWhitespace());
-
- // Something is wrong with the number of values that we read
- if (NUM_NET_VALUES > counters.size()) {
- KLOG_EVERY_N_SECS(WARNING, 60) << "Failed to parse enough values: expected "
- << NUM_NET_VALUES << " but found " << counters.size();
- return;
- }
-
- for (int i = 0; i < NUM_NET_VALUES; ++i) {
- int64_t* v = &(*result)[i];
- if (!safe_strto64(counters[i].as_string(), v)) {
- KLOG_EVERY_N_SECS(WARNING, 60) << "Failed to parse value: " << *v;
- *v = 0;
- }
- DCHECK_GE(*v, 0) << "Value " << i << ": " << *v;
- }
+ ReadIntArray(sp.substr(colon_idx + 1), NUM_NET_VALUES, result);
}
void SystemStateInfo::AddNetworkValues(const NetworkValues& a, NetworkValues* b) {
@@ -196,7 +202,7 @@ void SystemStateInfo::AddNetworkValues(const NetworkValues& a, NetworkValues* b)
}
void SystemStateInfo::ComputeNetworkUsage(int64_t period_ms) {
- if (period_ms == 0) return;
+ if (period_ms <= 0) return;
// Compute network throughput in bytes per second
const NetworkValues& cur = network_values_[net_val_idx_];
const NetworkValues& old = network_values_[1 - net_val_idx_];
@@ -207,4 +213,59 @@ void SystemStateInfo::ComputeNetworkUsage(int64_t period_ms) {
network_usage_.tx_rate = tx_bytes * 1000L / period_ms;
}
+void SystemStateInfo::ReadCurrentProcDiskStats() {
+ string path = "/proc/diskstats";
+ faststring buf;
+ if (!ReadFileOrWarn(path, &buf)) return;
+ ReadProcDiskStatsString(buf.ToString());
+}
+
+void SystemStateInfo::ReadProcDiskStatsString(const string& disk_stats) {
+ stringstream ss(disk_stats);
+ string line;
+
+ disk_val_idx_ = 1 - disk_val_idx_;
+ DCHECK(disk_val_idx_ == 0 || disk_val_idx_ == 1) << "disk_val_idx_: " << disk_val_idx_;
+ DiskValues& sum_values = disk_values_[disk_val_idx_];
+ memset(&sum_values, 0, sizeof(sum_values));
+
+ while (getline(ss, line)) {
+ DiskValues line_values;
+ ReadProcDiskStatsLine(line, &line_values);
+ AddDiskValues(line_values, &sum_values);
+ }
+}
+
+void SystemStateInfo::ReadProcDiskStatsLine(
+ const string& disk_stats, DiskValues* result) {
+ stringstream ss(disk_stats);
+
+ // Read the device IDs and name
+ string major, minor, disk_name;
+ ss >> major >> minor >> disk_name;
+
+ if (!DiskInfo::is_known_disk(disk_name)) {
+ memset(result, 0, sizeof(*result));
+ return;
+ }
+
+ ReadIntArray(StringPiece(disk_stats).substr(ss.tellg()), NUM_DISK_VALUES, result);
+}
+
+void SystemStateInfo::AddDiskValues(const DiskValues& a, DiskValues* b) {
+ for (int i = 0; i < NUM_DISK_VALUES; ++i) (*b)[i] += a[i];
+}
+
+void SystemStateInfo::ComputeDiskStats(int64_t period_ms) {
+ if (period_ms <= 0) return;
+ // Compute disk throughput in bytes per second
+ const DiskValues& cur = disk_values_[disk_val_idx_];
+ const DiskValues& old = disk_values_[1 - disk_val_idx_];
+ int64_t read_sectors = cur[DISK_SECTORS_READ] - old[DISK_SECTORS_READ];
+ disk_stats_.read_rate = read_sectors * BYTES_PER_SECTOR * 1000 / period_ms;
+
+ int64_t write_sectors = cur[DISK_SECTORS_WRITTEN] - old[DISK_SECTORS_WRITTEN];
+ disk_stats_.write_rate = write_sectors * BYTES_PER_SECTOR * 1000 / period_ms;
+}
+
} // namespace impala
diff --git a/be/src/util/system-state-info.h b/be/src/util/system-state-info.h
index e22d404..c17f562 100644
--- a/be/src/util/system-state-info.h
+++ b/be/src/util/system-state-info.h
@@ -20,10 +20,15 @@
#include <array>
#include <cstdint>
#include <string>
+#include <sstream>
#include <gtest/gtest_prod.h> // for FRIEND_TEST
#include "common/names.h"
+#include "common/logging.h"
+
+class StringPiece;
+
namespace impala {
/// Utility class to track and compute system resource consumption.
@@ -61,9 +66,17 @@ class SystemStateInfo {
/// calls to CaptureSystemStateSnapshot().
const NetworkUsage& GetNetworkUsage() { return network_usage_; }
- private:
- int64_t ParseInt64(const std::string& val) const;
+ /// Disk statistics rates in bytes per second
+ struct DiskStats {
+ int64_t read_rate;
+ int64_t write_rate;
+ };
+
+ /// Returns a struct containing the disk throughput for the interval between the last
+ /// two calls to CaptureSystemStateSnapshot().
+ const DiskStats& GetDiskStats() { return disk_stats_; }
+ private:
/// Rotates 'cpu_val_idx_' and reads the current CPU usage values from /proc/stat into
/// the current set of values.
void ReadCurrentProcStat();
@@ -126,7 +139,7 @@ class SystemStateInfo {
/// Two buffers to keep the current and previous set of network counter values.
NetworkValues network_values_[2];
/// Index into network_values_ that points to the current set of values. We maintain a
- /// separate index for CPU and network to be able to update them independently, e.g. in
+ /// separate index for network values to be able to update them independently, e.g. in
/// tests.
int net_val_idx_ = 0;
@@ -140,7 +153,7 @@ class SystemStateInfo {
/// Parses a single line as they appear in /proc/net/dev into 'result'. Entries are set
/// to 0 for the local loopback interface and for invalid entries.
- void ReadProcNetDevLine(const string& dev_string, NetworkValues* result);
+ void ReadProcNetDevLine(const StringPiece& dev_string, NetworkValues* result);
/// Computes b = b + a.
void AddNetworkValues(const NetworkValues& a, NetworkValues* b);
@@ -148,7 +161,7 @@ class SystemStateInfo {
/// Compute the network usage.
void ComputeNetworkUsage(int64_t period_ms);
- /// The compute network usage between the current and previous snapshots in
+ /// The network usage between the current and previous snapshots in
/// network_values_. Updated in ComputeNetworkUsage().
NetworkUsage network_usage_;
@@ -156,12 +169,71 @@ class SystemStateInfo {
/// CaptureSystemStateSnapshot().
int64_t last_net_update_ms_;
+ /// The enum names correspond to the fields of /proc/diskstats
+ /// https://www.kernel.org/doc/Documentation/iostats.txt
+ enum PROC_DISK_STAT_VALUES {
+ DISK_READS_COMPLETED,
+ DISK_READS_MERGED,
+ DISK_SECTORS_READ,
+ DISK_TIME_READING_MS,
+ DISK_WRITES_COMPLETED,
+ DISK_WRITES_MERGED,
+ DISK_SECTORS_WRITTEN,
+ DISK_TIME_WRITING_MS,
+ DISK_IO_IN_PROGRESS,
+ DISK_TIME_IN_IO_MS,
+ DISK_WEIGHTED_TIME_IN_IO_MS,
+ NUM_DISK_VALUES
+ };
+
+ /// Number of bytes per disk sector.
+ static const int BYTES_PER_SECTOR = 512;
+
+ /// We store these values in an array so that we can iterate over them, e.g. when
+ /// reading them from a file or summing them up.
+ typedef std::array<int64_t, NUM_DISK_VALUES> DiskValues;
+ /// Two buffers to keep the current and previous set of disk counter values.
+ DiskValues disk_values_[2];
+ /// Index into disk_values_ that points to the current set of values. We maintain a
+ /// separate index for disk values to be able to update them independently, e.g. in
+ /// tests.
+ int disk_val_idx_ = 0;
+
+ /// Rotates disk_val_idx_ and reads the current set of values from /proc/diskstats into
+ /// disk_values_.
+ void ReadCurrentProcDiskStats();
+
+ /// Rotates disk_val_idx_ and parses the content of 'disk_stats' into disk_values_.
+ /// disk_stats must be in the format of /proc/diskstats.
+ void ReadProcDiskStatsString(const string& disk_stats);
+
+ /// Parses a single line as they appear in /proc/diskstats into 'result'. Invalid
+ /// entries are set to 0.
+ void ReadProcDiskStatsLine(const string& disk_stats, DiskValues* result);
+
+ /// Computes b = b + a.
+ void AddDiskValues(const DiskValues& a, DiskValues* b);
+
+ /// Computes the read and write rate for the most recent period.
+ void ComputeDiskStats(int64_t period_ms);
+
+ /// The disk statistics between the current and previous snapshots in
+ /// disk_values_. Updated in ComputeDiskStats().
+ DiskStats disk_stats_;
+
+ /// The last time of reading disk stats values from /proc/diskstats. Used by
+ /// CaptureSystemStateSnapshot().
+ int64_t last_disk_update_ms_;
+
FRIEND_TEST(SystemStateInfoTest, ComputeCpuRatios);
FRIEND_TEST(SystemStateInfoTest, ComputeCpuRatiosIntOverflow);
+ FRIEND_TEST(SystemStateInfoTest, ComputeDiskStats);
FRIEND_TEST(SystemStateInfoTest, ComputeNetworkUsage);
+ FRIEND_TEST(SystemStateInfoTest, ParseProcDiskStatsString);
FRIEND_TEST(SystemStateInfoTest, ParseProcNetDevString);
FRIEND_TEST(SystemStateInfoTest, ParseProcNetDevStringCentos6);
FRIEND_TEST(SystemStateInfoTest, ParseProcStat);
+ FRIEND_TEST(SystemStateInfoTest, ReadProcDiskStats);
FRIEND_TEST(SystemStateInfoTest, ReadProcNetDev);
FRIEND_TEST(SystemStateInfoTest, ReadProcStat);
};
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index a1dca7f..fd4c796 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -441,6 +441,7 @@ class TestObservability(ImpalaTestSuite):
for line in profile.splitlines():
assert not re.search("HostCpu.*Percentage", line)
assert not re.search("HostNetworkRx", line)
+ assert not re.search("HostDiskReadThroughput", line)
def test_query_profile_contains_host_resource_metrics(self):
"""Tests that the query profile contains various CPU and network metrics."""
@@ -453,7 +454,9 @@ class TestObservability(ImpalaTestSuite):
"HostCpuSysPercentage (500.000ms):",
"HostCpuUserPercentage (500.000ms):",
"HostNetworkRx (500.000ms):",
- "HostNetworkTx (500.000ms):"]
+ "HostNetworkTx (500.000ms):",
+ "HostDiskReadThroughput (500.000ms):",
+ "HostDiskWriteThroughput (500.000ms):"]
# Assert that all expected counters exist in the profile.
for expected_str in expected_strs:
@@ -493,7 +496,7 @@ class TestObservability(ImpalaTestSuite):
result = self.execute_query("select sleep(2000)", query_opts)
thrift_profile = self._get_thrift_profile(result.query_id)
- expected_keys = ["HostCpuUserPercentage", "HostNetworkRx"]
+ expected_keys = ["HostCpuUserPercentage", "HostNetworkRx", "HostDiskReadThroughput"]
for key in expected_keys:
counters = self._find_ts_counters_in_thrift_profile(thrift_profile, key)
# The query will run on a single node, we will only find the counter once.