You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by aw...@apache.org on 2021/08/31 17:39:20 UTC
[kudu] branch master updated: KUDU-3204: Add a metric for amount of
available space
This is an automated email from the ASF dual-hosted git repository.
awong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new e60ac3c KUDU-3204: Add a metric for amount of available space
e60ac3c is described below
commit e60ac3ca93532fdffa71999e90661e42d7dcc0b0
Author: Abhishek Chennaka <ac...@cloudera.com>
AuthorDate: Fri Jul 23 14:03:31 2021 -0400
KUDU-3204: Add a metric for amount of available space
This commit adds two metrics to server entity.
- wal_dir_space_available_bytes - Exposes the free space available in the
wal directory
- data_dirs_space_available_bytes - Exposes the sum of free space in each
of the data directories.
We ensure the reserved space is being accounted for. This is specified by
the flags --fs_data_dirs_reserved_bytes and --fs_wal_dir_reserved_bytes in
data directories and wal directories respectively. For calculating the
total free space in data directories, we ensure we are not counting the
same device/filesystem twice by getting and checking for uniqueness
of the filesystemID of each of the data directories before summing it up.
Tested the uniqueness of the filesystemID by creating a new volume in my
local macOS machine. The space reported is based on the data directories in
different volumes (and hence unique filesystemID). Having multiple data
directories on the same volume does not increase the metric.
The free space calculated is cached for the period specified by the flags
--fs_data_dirs_available_space_cache_seconds and
--fs_wal_dir_available_space_cache_seconds.
Change-Id: I58a7419847d080498aaf431d1dab098e1af71ad0
Reviewed-on: http://gerrit.cloudera.org:8080/17725
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Tested-by: Kudu Jenkins
---
src/kudu/consensus/log.cc | 10 +--
src/kudu/fs/data_dirs.cc | 11 ++-
src/kudu/fs/fs_manager.cc | 10 +++
src/kudu/integration-tests/disk_failure-itest.cc | 48 ++++++++++++
src/kudu/server/server_base.cc | 96 ++++++++++++++++++++++++
src/kudu/server/server_base.h | 24 ++++++
src/kudu/util/env.h | 1 +
src/kudu/util/env_posix.cc | 1 +
8 files changed, 191 insertions(+), 10 deletions(-)
diff --git a/src/kudu/consensus/log.cc b/src/kudu/consensus/log.cc
index ea24555..716e7e5 100644
--- a/src/kudu/consensus/log.cc
+++ b/src/kudu/consensus/log.cc
@@ -151,15 +151,7 @@ TAG_FLAG(log_segment_size_bytes_for_tests, unsafe);
// Other flags.
// -----------------------------
-DEFINE_int64(fs_wal_dir_reserved_bytes, -1,
- "Number of bytes to reserve on the log directory filesystem for "
- "non-Kudu usage. The default, which is represented by -1, is that "
- "1% of the disk space on each disk will be reserved. Any other "
- "value specified represents the number of bytes reserved and must "
- "be greater than or equal to 0. Explicit percentages to reserve "
- "are not currently supported");
-DEFINE_validator(fs_wal_dir_reserved_bytes, [](const char* /*n*/, int64_t v) { return v >= -1; });
-TAG_FLAG(fs_wal_dir_reserved_bytes, runtime);
+DECLARE_int64(fs_wal_dir_reserved_bytes);
DEFINE_bool(fs_wal_use_file_cache, true,
"Whether to use the server-wide file cache for WAL segments and "
diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc
index f2f36be..b5dc54c 100644
--- a/src/kudu/fs/data_dirs.cc
+++ b/src/kudu/fs/data_dirs.cc
@@ -77,10 +77,19 @@ TAG_FLAG(fs_data_dirs_reserved_bytes, runtime);
TAG_FLAG(fs_data_dirs_reserved_bytes, evolving);
DEFINE_int32(fs_data_dirs_available_space_cache_seconds, 10,
- "Number of seconds we cache the available disk space in the block manager. ");
+ "Number of seconds we cache the available disk space in the block manager.");
+DEFINE_validator(fs_data_dirs_available_space_cache_seconds,
+ [](const char* /*n*/, int32_t v) { return v >= 0; });
TAG_FLAG(fs_data_dirs_available_space_cache_seconds, advanced);
TAG_FLAG(fs_data_dirs_available_space_cache_seconds, evolving);
+DEFINE_int32(fs_wal_dir_available_space_cache_seconds, 10,
+ "Number of seconds we cache the available disk space the WAL directory.");
+DEFINE_validator(fs_wal_dir_available_space_cache_seconds,
+ [](const char* /*n*/, int32_t v) { return v >= 0; });
+TAG_FLAG(fs_wal_dir_available_space_cache_seconds, advanced);
+TAG_FLAG(fs_wal_dir_available_space_cache_seconds, evolving);
+
DEFINE_bool(fs_lock_data_dirs, true,
"Lock the data directories to prevent concurrent usage. "
"Note that read-only concurrent usage is still allowed.");
diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc
index a628469..a60ef1a 100644
--- a/src/kudu/fs/fs_manager.cc
+++ b/src/kudu/fs/fs_manager.cc
@@ -98,6 +98,16 @@ DEFINE_string(fs_metadata_dir, "",
"will be used as the metadata directory.");
TAG_FLAG(fs_metadata_dir, stable);
+DEFINE_int64(fs_wal_dir_reserved_bytes, -1,
+ "Number of bytes to reserve on the log directory filesystem for "
+ "non-Kudu usage. The default, which is represented by -1, is that "
+ "1% of the disk space on each disk will be reserved. Any other "
+ "value specified represents the number of bytes reserved and must "
+ "be greater than or equal to 0. Explicit percentages to reserve "
+ "are not currently supported");
+DEFINE_validator(fs_wal_dir_reserved_bytes, [](const char* /*n*/, int64_t v) { return v >= -1; });
+TAG_FLAG(fs_wal_dir_reserved_bytes, runtime);
+
using kudu::fs::BlockManagerOptions;
using kudu::fs::CreateBlockOptions;
using kudu::fs::DataDirManager;
diff --git a/src/kudu/integration-tests/disk_failure-itest.cc b/src/kudu/integration-tests/disk_failure-itest.cc
index 7303ebf..ec84c06 100644
--- a/src/kudu/integration-tests/disk_failure-itest.cc
+++ b/src/kudu/integration-tests/disk_failure-itest.cc
@@ -54,6 +54,8 @@ METRIC_DECLARE_entity(tablet);
METRIC_DECLARE_gauge_size(num_rowsets_on_disk);
METRIC_DECLARE_gauge_uint64(data_dirs_failed);
METRIC_DECLARE_gauge_uint32(tablets_num_failed);
+METRIC_DECLARE_gauge_uint64(wal_dir_space_available_bytes);
+METRIC_DECLARE_gauge_uint64(data_dirs_space_available_bytes);
using kudu::client::sp::shared_ptr;
using kudu::client::KuduClient;
@@ -318,6 +320,52 @@ TEST_P(TabletServerDiskErrorITest, TestFailOnBootstrap) {
NO_FATALS(v.CheckCluster());
};
+TEST_P(TabletServerDiskErrorITest, TestSpaceAvailableMetrics) {
+ // Get the wal_dir_space_available_bytes, data_dirs_space_available_bytes and make sure
+ // they are not -1. We could do ASSERT_EQ of the metrics and the current available
+ // space but there will most likely be some data written between the two calls causing
+ // a failure. Hence checking if the values are more than the 0. We fetch the metric
+ // twice and ensure it is being cached for 10 seconds by default specified by the
+ // flags --fs_wal_dir_available_space_cache_seconds and
+ // --fs_data_dirs_available_space_cache_seconds.
+ const auto get_metrics = [&] (int64_t* wal_dir_space, int64_t* data_dir_space) {
+ RETURN_NOT_OK(itest::GetInt64Metric(cluster_->tablet_server(0)->bound_http_hostport(),
+ &METRIC_ENTITY_server, nullptr,
+ &METRIC_wal_dir_space_available_bytes, "value",
+ wal_dir_space));
+ return itest::GetInt64Metric(cluster_->tablet_server(0)->bound_http_hostport(),
+ &METRIC_ENTITY_server, nullptr,
+ &METRIC_data_dirs_space_available_bytes, "value",
+ data_dir_space);
+ };
+ int64_t wal_dir_space;
+ int64_t data_dir_space;
+ int64_t wal_dir_space_refetch;
+ int64_t data_dir_space_refetch;
+ ASSERT_OK(get_metrics(&wal_dir_space, &data_dir_space));
+ ASSERT_GT(wal_dir_space, 0);
+ ASSERT_GT(data_dir_space, 0);
+ ASSERT_OK(get_metrics(&wal_dir_space_refetch, &data_dir_space_refetch));
+ ASSERT_EQ(wal_dir_space, wal_dir_space_refetch);
+ ASSERT_EQ(data_dir_space, data_dir_space_refetch);
+
+ ExternalTabletServer* error_ts = cluster_->tablet_server(0);
+
+ // Inject EIO into one of the data directories and check if data_dirs_space_available_bytes
+ // now equals to -1
+ error_ts->mutable_flags()->emplace_back(
+ Substitute("--env_inject_eio_globs=$0", JoinPathSegments(error_ts->data_dirs()[1],
+ "**")));
+ error_ts->mutable_flags()->emplace_back("--env_inject_eio=1.0");
+
+ error_ts->Shutdown();
+ ASSERT_OK(error_ts->Restart());
+
+ ASSERT_OK(get_metrics(&wal_dir_space, &data_dir_space));
+ ASSERT_NE(wal_dir_space, -1);
+ ASSERT_EQ(data_dir_space, -1);
+};
+
TEST_P(TabletServerDiskErrorITest, TestFailDuringScanWorkload) {
// Set up a workload that only reads from the tablets.
TestWorkload read(cluster_.get());
diff --git a/src/kudu/server/server_base.cc b/src/kudu/server/server_base.cc
index 0cba6fc..e255076 100644
--- a/src/kudu/server/server_base.cc
+++ b/src/kudu/server/server_base.cc
@@ -21,6 +21,7 @@
#include <cstdint>
#include <functional>
#include <mutex>
+#include <set>
#include <sstream>
#include <string>
#include <vector>
@@ -40,6 +41,7 @@
#include "kudu/fs/fs_manager.h"
#include "kudu/fs/fs_report.h"
#include "kudu/gutil/integral_types.h"
+#include "kudu/gutil/map-util.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/strings/numbers.h"
#include "kudu/gutil/strings/split.h"
@@ -240,6 +242,10 @@ DECLARE_bool(use_hybrid_clock);
DECLARE_int32(dns_resolver_max_threads_num);
DECLARE_uint32(dns_resolver_cache_capacity_mb);
DECLARE_uint32(dns_resolver_cache_ttl_sec);
+DECLARE_int32(fs_data_dirs_available_space_cache_seconds);
+DECLARE_int32(fs_wal_dir_available_space_cache_seconds);
+DECLARE_int64(fs_wal_dir_reserved_bytes);
+DECLARE_int64(fs_data_dirs_reserved_bytes);
DECLARE_string(log_filename);
DECLARE_string(keytab_file);
DECLARE_string(principal);
@@ -250,6 +256,18 @@ METRIC_DEFINE_gauge_int64(server, uptime,
kudu::MetricUnit::kMicroseconds,
"Time interval since the server has started",
kudu::MetricLevel::kInfo);
+METRIC_DEFINE_gauge_int64(server, wal_dir_space_available_bytes,
+ "WAL Directory Space Free",
+ kudu::MetricUnit::kBytes,
+ "Total WAL directory space available. Set to "
+ "-1 if reading the disk fails",
+ kudu::MetricLevel::kInfo);
+METRIC_DEFINE_gauge_int64(server, data_dirs_space_available_bytes,
+ "Data Directories Space Free",
+ kudu::MetricUnit::kBytes,
+ "Total space available in all the data directories. Set to "
+ "-1 if reading any of the disks fails",
+ kudu::MetricLevel::kInfo);
using kudu::security::RpcAuthentication;
using kudu::security::RpcEncryption;
@@ -413,6 +431,19 @@ shared_ptr<MemTracker> CreateMemTrackerForServer() {
return shared_ptr<MemTracker>(MemTracker::CreateTracker(-1, id_str));
}
+// Calculates the free space on the given WAL/data directory's disk. Returns -1 in case of disk
+// failure.
+inline int64_t CalculateAvailableSpace(const ServerBaseOptions& options, const string& dir,
+ int64_t flag_reserved_bytes, SpaceInfo* space_info) {
+ if (!options.env->GetSpaceInfo(dir, space_info).ok()) {
+ return -1;
+ }
+ bool should_reserve_one_percent = flag_reserved_bytes == -1;
+ int reserved_bytes = should_reserve_one_percent ?
+ space_info->capacity_bytes / 100 : flag_reserved_bytes;
+ return std::max(static_cast<int64_t>(0), space_info->free_bytes - reserved_bytes);
+}
+
int64_t GetFileCacheCapacity(Env* env) {
// Maximize this process' open file limit first, if possible.
static std::once_flag once;
@@ -864,11 +895,29 @@ Status ServerBase::Start() {
// information ready at this point.
start_time_ = MonoTime::Now();
start_walltime_ = static_cast<int64_t>(WallTime_Now());
+ wal_dir_space_last_check_ = MonoTime::Min();
+ wal_dir_available_space_ = -1;
+ data_dirs_space_last_check_ = MonoTime::Min();
+ data_dirs_available_space_ = -1;
METRIC_uptime.InstantiateFunctionGauge(
metric_entity_,
[this]() {return (MonoTime::Now() - this->start_time()).ToMicroseconds();})->
AutoDetachToLastValue(&metric_detacher_);
+ METRIC_data_dirs_space_available_bytes.InstantiateFunctionGauge(
+ metric_entity_,
+ [this]() {
+ return RefreshDataDirAvailableSpaceIfExpired(options_, *fs_manager_);
+ })
+ ->AutoDetachToLastValue(&metric_detacher_);
+
+ METRIC_wal_dir_space_available_bytes.InstantiateFunctionGauge(
+ metric_entity_,
+ [this]() {
+ return RefreshWalDirAvailableSpaceIfExpired(options_, *fs_manager_);
+ })
+ ->AutoDetachToLastValue(&metric_detacher_);
+
RETURN_NOT_OK_PREPEND(StartMetricsLogging(), "Could not enable metrics logging");
@@ -897,6 +946,53 @@ Status ServerBase::Start() {
return Status::OK();
}
+int64_t ServerBase::RefreshWalDirAvailableSpaceIfExpired(const ServerBaseOptions& options,
+ const FsManager& fs_manager) {
+ {
+ std::lock_guard<simple_spinlock> l(lock_);
+ if (MonoTime::Now() < wal_dir_space_last_check_ + MonoDelta::FromSeconds(
+ FLAGS_fs_wal_dir_available_space_cache_seconds))
+ return wal_dir_available_space_;
+ }
+ SpaceInfo space_info_waldir;
+ int64_t wal_dir_space = CalculateAvailableSpace(options, fs_manager.GetWalsRootDir(),
+ FLAGS_fs_wal_dir_reserved_bytes,
+ &space_info_waldir);
+ std::lock_guard<simple_spinlock> l(lock_);
+ wal_dir_space_last_check_ = MonoTime::Now();
+ wal_dir_available_space_ = wal_dir_space;
+ return wal_dir_available_space_;
+}
+
+int64_t ServerBase::RefreshDataDirAvailableSpaceIfExpired(const ServerBaseOptions& options,
+ const FsManager& fs_manager) {
+ {
+ std::lock_guard<simple_spinlock> l(lock_);
+ if (MonoTime::Now() < data_dirs_space_last_check_ + MonoDelta::FromSeconds(
+ FLAGS_fs_data_dirs_available_space_cache_seconds))
+ return data_dirs_available_space_;
+ }
+ SpaceInfo space_info_datadir;
+ std::set<int64_t> fs_id;
+ int64_t data_dirs_available_space = 0;
+ for (const string& data_dir : fs_manager.GetDataRootDirs()) {
+ int64_t available_space = CalculateAvailableSpace(options, data_dir,
+ FLAGS_fs_data_dirs_reserved_bytes,
+ &space_info_datadir);
+ if (available_space == -1) {
+ data_dirs_available_space = -1;
+ break;
+ }
+ if (InsertIfNotPresent(&fs_id, space_info_datadir.filesystem_id)) {
+ data_dirs_available_space += available_space;
+ }
+ }
+ std::lock_guard<simple_spinlock> l(lock_);
+ data_dirs_space_last_check_ = MonoTime::Now();
+ data_dirs_available_space_ = data_dirs_available_space;
+ return data_dirs_available_space_;
+}
+
void ServerBase::UnregisterAllServices() {
messenger_->UnregisterAllServices();
}
diff --git a/src/kudu/server/server_base.h b/src/kudu/server/server_base.h
index ce8b7c6..a8a16e9 100644
--- a/src/kudu/server/server_base.h
+++ b/src/kudu/server/server_base.h
@@ -28,6 +28,7 @@
#include "kudu/security/simple_acl.h"
#include "kudu/server/server_base_options.h"
#include "kudu/util/countdown_latch.h"
+#include "kudu/util/locks.h"
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
#include "kudu/util/status.h"
@@ -246,6 +247,20 @@ class ServerBase {
void TcmallocMemoryGcThread();
#endif
+ // The free space on the WAL disk is updated if expired and stored in 'wal_dir_available_space_'
+ // and returned. In case of errors during disk reading, it is set to -1. The
+ // cache period is specified by the flag --fs_wal_dir_available_space_cache_second (Defaults
+ // to 10 seconds).
+ int64_t RefreshWalDirAvailableSpaceIfExpired(const ServerBaseOptions& options,
+ const FsManager& fs_manager);
+
+ // The total free space on all the data directories is updated if expired and stored in
+ // 'data_dirs_available_space_' and returned. It accounts for multiple directories on a single
+ // disk. In case of errors during disk reading, it is set ot -1. The cache period is specified
+ // by the flag --fs_data_dirs_available_space_cache_seconds (Defaults to 10 seconds).
+ int64_t RefreshDataDirAvailableSpaceIfExpired(const ServerBaseOptions& options,
+ const FsManager& fs_manager);
+
// Utility object for DNS name resolutions.
std::unique_ptr<DnsResolver> dns_resolver_;
@@ -260,11 +275,20 @@ class ServerBase {
std::unique_ptr<ScopedGLogMetrics> glog_metrics_;
+ simple_spinlock lock_;
+
+
+ int64_t wal_dir_available_space_;
+ int64_t data_dirs_available_space_;
+ MonoTime wal_dir_space_last_check_;
+ MonoTime data_dirs_space_last_check_;
+
// NOTE: it's important that this is the first member to be destructed. This
// ensures we do not attempt to collect metrics while calling the destructor.
FunctionGaugeDetacher metric_detacher_;
DISALLOW_COPY_AND_ASSIGN(ServerBase);
+
};
} // namespace server
diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h
index 611de80..ce436a8 100644
--- a/src/kudu/util/env.h
+++ b/src/kudu/util/env.h
@@ -45,6 +45,7 @@ class ArrayView;
struct SpaceInfo {
int64_t capacity_bytes; // Capacity of a filesystem, in bytes.
int64_t free_bytes; // Bytes available to non-privileged processes.
+ uint64_t filesystem_id; // FilesystemID returned by statvfs()
};
class Env {
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index e35f968..b25e9c5 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -1443,6 +1443,7 @@ class PosixEnv : public Env {
RETURN_NOT_OK(StatVfs(path, &buf));
space_info->capacity_bytes = buf.f_frsize * buf.f_blocks;
space_info->free_bytes = buf.f_frsize * buf.f_bavail;
+ space_info->filesystem_id = buf.f_fsid;
return Status::OK();
}