You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2026/08/08 16:23:29 UTC
(kudu) branch master updated: KUDU-3362: expose per-tablet stats via table statistics
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 4bcd631c4 KUDU-3362: expose per-tablet stats via table statistics
4bcd631c4 is described below
commit 4bcd631c449d3c6649730ca6e07660c8168f0d44
Author: Yan-Daojiang <ya...@163.com>
AuthorDate: Tue Jun 30 19:49:05 2026 +0800
KUDU-3362: expose per-tablet stats via table statistics
Add per-tablet statistics to the GetTableStatistics response and surface
them in kudu table statistics --show_tablets.
Default table-level output is unchanged when --show_tablets is not set.
Change-Id: If7889669181633a762c614afe96a9ea0827ed7f1
Reviewed-on: http://gerrit.cloudera.org:8080/24650
Reviewed-by: Marton Greber <gr...@gmail.com>
Tested-by: Marton Greber <gr...@gmail.com>
Reviewed-by: Alexey Serbin <al...@apache.org>
---
src/kudu/master/catalog_manager.cc | 22 +++++++++++
src/kudu/master/master-test.cc | 76 ++++++++++++++++++++++++++++++++++++
src/kudu/master/master.proto | 16 ++++++++
src/kudu/tools/kudu-tool-test.cc | 19 +++++++++
src/kudu/tools/tool_action_common.cc | 12 ++++++
src/kudu/tools/tool_action_table.cc | 61 ++++++++++++++++++++++++++---
6 files changed, 200 insertions(+), 6 deletions(-)
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 04476f274..d29383626 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -4354,6 +4354,28 @@ Status CatalogManager::GetTableStatistics(const GetTableStatisticsRequestPB* req
resp->set_row_count_limit(TableInfo::TABLE_WRITE_DEFAULT_LIMIT);
}
}
+
+ if (req->include_tablet_statistics()) {
+ const auto tablet_map = table->tablet_map();
+ for (const auto& entry : tablet_map) {
+ const auto& tablet = entry.second;
+ TabletMetadataLock tablet_lock(tablet.get(), LockMode::READ);
+ if (tablet_lock.data().is_deleted()) {
+ continue;
+ }
+
+ auto* tablet_stats = resp->add_tablet_statistics();
+ tablet_stats->set_tablet_id(tablet->id());
+ const auto stats = tablet->GetStats();
+ if (stats.has_on_disk_size()) {
+ tablet_stats->set_on_disk_size(stats.on_disk_size());
+ }
+ if (stats.has_live_row_count()) {
+ tablet_stats->set_live_row_count(stats.live_row_count());
+ }
+ }
+ }
+
return Status::OK();
}
diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc
index 934ab749f..23b452192 100644
--- a/src/kudu/master/master-test.cc
+++ b/src/kudu/master/master-test.cc
@@ -3646,6 +3646,82 @@ TEST_F(MasterTest, TestTableStatisticsWithOldVersion) {
}
}
+TEST_F(MasterTest, TestGetTableStatisticsContainsTabletStatistics) {
+ const char* kTableName = "testtable_tablet_statistics";
+ const Schema kTableSchema({ ColumnSchema("key", INT32) }, 1);
+ ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+ vector<scoped_refptr<TableInfo>> tables;
+ {
+ CatalogManager::ScopedLeaderSharedLock l(master_->catalog_manager());
+ master_->catalog_manager()->GetAllTables(&tables);
+ ASSERT_EQ(1, tables.size());
+ }
+ const auto& table = tables[0];
+ vector<scoped_refptr<TabletInfo>> tablets;
+ table->GetAllTablets(&tablets);
+ ASSERT_EQ(3, tablets.size());
+
+ {
+ tablet::ReportedTabletStatsPB stats;
+ stats.set_on_disk_size(1024);
+ stats.set_live_row_count(10);
+ tablets[0]->UpdateStats(stats);
+ }
+ {
+ tablet::ReportedTabletStatsPB stats;
+ stats.set_on_disk_size(2048);
+ tablets[1]->UpdateStats(stats);
+ }
+
+ // By default per-tablet statistics are not collected.
+ {
+ GetTableStatisticsRequestPB req;
+ req.mutable_table()->set_table_name(kTableName);
+ GetTableStatisticsResponsePB resp;
+ RpcController rpc;
+ ASSERT_OK(proxy_->GetTableStatistics(req, &resp, &rpc));
+ ASSERT_FALSE(resp.has_error()) << SecureDebugString(resp);
+ ASSERT_EQ(0, resp.tablet_statistics_size());
+ }
+
+ GetTableStatisticsRequestPB req;
+ req.mutable_table()->set_table_name(kTableName);
+ req.set_include_tablet_statistics(true);
+ GetTableStatisticsResponsePB resp;
+ RpcController rpc;
+ ASSERT_OK(proxy_->GetTableStatistics(req, &resp, &rpc));
+ ASSERT_FALSE(resp.has_error()) << SecureDebugString(resp);
+
+ unordered_map<string, const TabletStatisticsPB*> stats_by_tablet_id;
+ for (const auto& tablet_stats : resp.tablet_statistics()) {
+ stats_by_tablet_id.emplace(tablet_stats.tablet_id(), &tablet_stats);
+ }
+ ASSERT_EQ(tablets.size(), stats_by_tablet_id.size());
+
+ {
+ const auto* stats = FindOrNull(stats_by_tablet_id, tablets[0]->id());
+ ASSERT_NE(nullptr, stats);
+ ASSERT_TRUE((*stats)->has_on_disk_size());
+ ASSERT_TRUE((*stats)->has_live_row_count());
+ ASSERT_EQ(1024, (*stats)->on_disk_size());
+ ASSERT_EQ(10, (*stats)->live_row_count());
+ }
+ {
+ const auto* stats = FindOrNull(stats_by_tablet_id, tablets[1]->id());
+ ASSERT_NE(nullptr, stats);
+ ASSERT_TRUE((*stats)->has_on_disk_size());
+ ASSERT_FALSE((*stats)->has_live_row_count());
+ ASSERT_EQ(2048, (*stats)->on_disk_size());
+ }
+ {
+ const auto* stats = FindOrNull(stats_by_tablet_id, tablets[2]->id());
+ ASSERT_NE(nullptr, stats);
+ ASSERT_FALSE((*stats)->has_on_disk_size());
+ ASSERT_FALSE((*stats)->has_live_row_count());
+ }
+}
+
TEST_F(MasterTest, TestDeletedTablesAndTabletsCleanup) {
FLAGS_enable_metadata_cleanup_for_deleted_tables_and_tablets = true;
FLAGS_metadata_for_deleted_table_and_tablet_reserved_secs = 1;
diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto
index 12b74f4d6..fc18bd54d 100644
--- a/src/kudu/master/master.proto
+++ b/src/kudu/master/master.proto
@@ -690,6 +690,19 @@ message ListTablesResponsePB {
message GetTableStatisticsRequestPB {
required TableIdentifierPB table = 1;
+
+ // Whether to include per-tablet statistics in the response. Disabled by
+ // default to avoid the cost of collecting them when they are not needed.
+ optional bool include_tablet_statistics = 2;
+}
+
+message TabletStatisticsPB {
+ optional bytes tablet_id = 1;
+
+ // The on-disk size and live row count come from the latest leader tablet
+ // report. They are left unset if no leader has reported for the tablet yet.
+ optional int64 on_disk_size = 2;
+ optional int64 live_row_count = 3;
}
message GetTableStatisticsResponsePB {
@@ -703,6 +716,9 @@ message GetTableStatisticsResponsePB {
// The table limit
optional int64 disk_size_limit = 4;
optional int64 row_count_limit = 5;
+
+ // Per-tablet statistics.
+ repeated TabletStatisticsPB tablet_statistics = 6;
}
// This data structure is used to specify a table's partition key.
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index ef00bd516..ec9961eb6 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -10036,6 +10036,25 @@ TEST_F(ToolTest, TestGetTableStatisticsNotSupported) {
ASSERT_STR_CONTAINS(stdout, "live row count: N/A");
}
+TEST_F(ToolTest, TestGetTableStatisticsShowTablets) {
+ NO_FATALS(StartExternalMiniCluster());
+
+ TestWorkload workload(cluster_.get());
+ workload.set_num_replicas(1);
+ workload.Setup();
+
+ string stdout;
+ NO_FATALS(RunActionStdoutString(
+ Substitute("table statistics $0 $1 --show_tablets",
+ cluster_->master()->bound_rpc_addr().ToString(),
+ TestWorkload::kDefaultTableName),
+ &stdout));
+ ASSERT_STR_CONTAINS(stdout, string("TABLE ") + TestWorkload::kDefaultTableName);
+ ASSERT_STR_CONTAINS(stdout, "tablet_id");
+ ASSERT_STR_CONTAINS(stdout, "on_disk_size");
+ ASSERT_STR_CONTAINS(stdout, "live_row_count");
+}
+
// Run one of the cluster-targeted subcommands verifying the functionality of
// the --connection_negotiation_timeout command-line option (the RPC
// connection negotiation timeout).
diff --git a/src/kudu/tools/tool_action_common.cc b/src/kudu/tools/tool_action_common.cc
index 97e2ea335..0a0881a7b 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ b/src/kudu/tools/tool_action_common.cc
@@ -1342,5 +1342,17 @@ Status LeaderMasterProxy::SyncRpc(
const ResponseCallback&)>& func,
std::vector<uint32_t> required_feature_flags);
+template
+Status LeaderMasterProxy::SyncRpc(
+ const master::GetTableStatisticsRequestPB& req,
+ master::GetTableStatisticsResponsePB* resp,
+ string func_name,
+ const std::function<void(MasterServiceProxy*,
+ const master::GetTableStatisticsRequestPB&,
+ master::GetTableStatisticsResponsePB*,
+ RpcController*,
+ const ResponseCallback&)>& func,
+ std::vector<uint32_t> required_feature_flags);
+
} // namespace tools
} // namespace kudu
diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc
index a6bab8cfb..868cfd0cc 100644
--- a/src/kudu/tools/tool_action_table.cc
+++ b/src/kudu/tools/tool_action_table.cc
@@ -52,6 +52,7 @@
#include "kudu/common/wire_protocol.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/stl_util.h"
+#include "kudu/gutil/strings/human_readable.h"
#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/numbers.h"
#include "kudu/gutil/strings/split.h"
@@ -91,6 +92,8 @@ using kudu::client::KuduTableStatistics;
using kudu::client::KuduValue;
using kudu::client::internal::ReplicaController;
using kudu::iequals;
+using kudu::master::GetTableStatisticsRequestPB;
+using kudu::master::GetTableStatisticsResponsePB;
using kudu::master::ListInFlightTablesRequestPB;
using kudu::master::ListInFlightTablesResponsePB;
using kudu::master::MasterServiceProxy;
@@ -127,6 +130,9 @@ DEFINE_bool(list_tablets, false,
DEFINE_bool(show_table_info, false,
"Include extra information such as number of tablets, replicas, "
"and live row count for a table in the output");
+DEFINE_bool(show_tablets, false,
+ "Also list per-tablet on-disk size and live row count under the "
+ "table-level statistics output.");
DEFINE_bool(show_tablet_partition_info, false,
"Include partition keys information corresponding to tablet in the output.");
@@ -1598,13 +1604,55 @@ Status GetTableStatistics(const RunnerContext& context) {
client::sp::shared_ptr<KuduClient> client;
RETURN_NOT_OK(CreateKuduClient(context, &client));
- unique_ptr<KuduTableStatistics> statistics;
- KuduTableStatistics *table_statistics;
- RETURN_NOT_OK_PREPEND(client->GetTableStatistics(table_name, &table_statistics),
- "failed to get table statistics.");
- statistics.reset(table_statistics);
+ // Use the client API when per-tablet statistics aren't requested.
+ if (!FLAGS_show_tablets) {
+ unique_ptr<KuduTableStatistics> statistics;
+ KuduTableStatistics* table_statistics;
+ RETURN_NOT_OK_PREPEND(client->GetTableStatistics(table_name, &table_statistics),
+ "failed to get table statistics.");
+ statistics.reset(table_statistics);
+ cout << "TABLE " << table_name << endl;
+ cout << statistics->ToString() << endl;
+ return Status::OK();
+ }
+
+ // Otherwise a single RPC returns both the table-level and per-tablet
+ // statistics, avoiding a redundant round-trip.
+ LeaderMasterProxy proxy;
+ RETURN_NOT_OK(proxy.Init(context));
+
+ GetTableStatisticsRequestPB req;
+ GetTableStatisticsResponsePB resp;
+ req.mutable_table()->set_table_name(table_name);
+ req.set_include_tablet_statistics(true);
+ RETURN_NOT_OK((proxy.SyncRpc<GetTableStatisticsRequestPB, GetTableStatisticsResponsePB>(
+ req, &resp, "GetTableStatistics", &MasterServiceProxy::GetTableStatisticsAsync)));
+
+ if (resp.has_error()) {
+ return StatusFromPB(resp.error().status());
+ }
+
cout << "TABLE " << table_name << endl;
- cout << statistics->ToString() << endl;
+ cout << Substitute("on disk size: $0\n"
+ "live row count: $1\n"
+ "on disk size limit: $2\n"
+ "live row count limit: $3\n",
+ resp.has_on_disk_size() ? std::to_string(resp.on_disk_size()) : "N/A",
+ resp.has_live_row_count() ? std::to_string(resp.live_row_count()) : "N/A",
+ resp.has_disk_size_limit() ? std::to_string(resp.disk_size_limit()) : "N/A",
+ resp.has_row_count_limit() ? std::to_string(resp.row_count_limit()) : "N/A")
+ << endl;
+
+ cout << "TABLETS" << endl;
+ DataTable data_table({ "tablet_id", "on_disk_size", "live_row_count" });
+ for (const auto& t : resp.tablet_statistics()) {
+ data_table.AddRow({
+ t.tablet_id(),
+ t.has_on_disk_size() ? HumanReadableNumBytes::ToString(t.on_disk_size()) : "N/A",
+ t.has_live_row_count() ? std::to_string(t.live_row_count()) : "N/A",
+ });
+ }
+ RETURN_NOT_OK(data_table.PrintTo(cout));
return Status::OK();
}
@@ -2238,6 +2286,7 @@ unique_ptr<Mode> BuildTableMode() {
ClusterActionBuilder("statistics", &GetTableStatistics)
.Description("Get table statistics")
.AddRequiredParameter({ kTableNameArg, "Name of the table to get statistics" })
+ .AddOptionalParameter("show_tablets")
.Build();
unique_ptr<Action> create_table =