You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2026/08/08 16:23:29 UTC

(kudu) branch master updated: KUDU-3362: expose per-tablet stats via table statistics

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new 4bcd631c4 KUDU-3362: expose per-tablet stats via table statistics
4bcd631c4 is described below

commit 4bcd631c449d3c6649730ca6e07660c8168f0d44
Author: Yan-Daojiang <ya...@163.com>
AuthorDate: Tue Jun 30 19:49:05 2026 +0800

    KUDU-3362: expose per-tablet stats via table statistics
    
    Add per-tablet statistics to the GetTableStatistics response and surface
    them in kudu table statistics --show_tablets.
    
    Default table-level output is unchanged when --show_tablets is not set.
    
    Change-Id: If7889669181633a762c614afe96a9ea0827ed7f1
    Reviewed-on: http://gerrit.cloudera.org:8080/24650
    Reviewed-by: Marton Greber <gr...@gmail.com>
    Tested-by: Marton Greber <gr...@gmail.com>
    Reviewed-by: Alexey Serbin <al...@apache.org>
---
 src/kudu/master/catalog_manager.cc   | 22 +++++++++++
 src/kudu/master/master-test.cc       | 76 ++++++++++++++++++++++++++++++++++++
 src/kudu/master/master.proto         | 16 ++++++++
 src/kudu/tools/kudu-tool-test.cc     | 19 +++++++++
 src/kudu/tools/tool_action_common.cc | 12 ++++++
 src/kudu/tools/tool_action_table.cc  | 61 ++++++++++++++++++++++++++---
 6 files changed, 200 insertions(+), 6 deletions(-)

diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 04476f274..d29383626 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -4354,6 +4354,28 @@ Status CatalogManager::GetTableStatistics(const GetTableStatisticsRequestPB* req
       resp->set_row_count_limit(TableInfo::TABLE_WRITE_DEFAULT_LIMIT);
     }
   }
+
+  if (req->include_tablet_statistics()) {
+    const auto tablet_map = table->tablet_map();
+    for (const auto& entry : tablet_map) {
+      const auto& tablet = entry.second;
+      TabletMetadataLock tablet_lock(tablet.get(), LockMode::READ);
+      if (tablet_lock.data().is_deleted()) {
+        continue;
+      }
+
+      auto* tablet_stats = resp->add_tablet_statistics();
+      tablet_stats->set_tablet_id(tablet->id());
+      const auto stats = tablet->GetStats();
+      if (stats.has_on_disk_size()) {
+        tablet_stats->set_on_disk_size(stats.on_disk_size());
+      }
+      if (stats.has_live_row_count()) {
+        tablet_stats->set_live_row_count(stats.live_row_count());
+      }
+    }
+  }
+
   return Status::OK();
 }
 
diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc
index 934ab749f..23b452192 100644
--- a/src/kudu/master/master-test.cc
+++ b/src/kudu/master/master-test.cc
@@ -3646,6 +3646,82 @@ TEST_F(MasterTest, TestTableStatisticsWithOldVersion) {
   }
 }
 
+TEST_F(MasterTest, TestGetTableStatisticsContainsTabletStatistics) {
+  const char* kTableName = "testtable_tablet_statistics";
+  const Schema kTableSchema({ ColumnSchema("key", INT32) }, 1);
+  ASSERT_OK(CreateTable(kTableName, kTableSchema));
+
+  vector<scoped_refptr<TableInfo>> tables;
+  {
+    CatalogManager::ScopedLeaderSharedLock l(master_->catalog_manager());
+    master_->catalog_manager()->GetAllTables(&tables);
+    ASSERT_EQ(1, tables.size());
+  }
+  const auto& table = tables[0];
+  vector<scoped_refptr<TabletInfo>> tablets;
+  table->GetAllTablets(&tablets);
+  ASSERT_EQ(3, tablets.size());
+
+  {
+    tablet::ReportedTabletStatsPB stats;
+    stats.set_on_disk_size(1024);
+    stats.set_live_row_count(10);
+    tablets[0]->UpdateStats(stats);
+  }
+  {
+    tablet::ReportedTabletStatsPB stats;
+    stats.set_on_disk_size(2048);
+    tablets[1]->UpdateStats(stats);
+  }
+
+  // By default per-tablet statistics are not collected.
+  {
+    GetTableStatisticsRequestPB req;
+    req.mutable_table()->set_table_name(kTableName);
+    GetTableStatisticsResponsePB resp;
+    RpcController rpc;
+    ASSERT_OK(proxy_->GetTableStatistics(req, &resp, &rpc));
+    ASSERT_FALSE(resp.has_error()) << SecureDebugString(resp);
+    ASSERT_EQ(0, resp.tablet_statistics_size());
+  }
+
+  GetTableStatisticsRequestPB req;
+  req.mutable_table()->set_table_name(kTableName);
+  req.set_include_tablet_statistics(true);
+  GetTableStatisticsResponsePB resp;
+  RpcController rpc;
+  ASSERT_OK(proxy_->GetTableStatistics(req, &resp, &rpc));
+  ASSERT_FALSE(resp.has_error()) << SecureDebugString(resp);
+
+  unordered_map<string, const TabletStatisticsPB*> stats_by_tablet_id;
+  for (const auto& tablet_stats : resp.tablet_statistics()) {
+    stats_by_tablet_id.emplace(tablet_stats.tablet_id(), &tablet_stats);
+  }
+  ASSERT_EQ(tablets.size(), stats_by_tablet_id.size());
+
+  {
+    const auto* stats = FindOrNull(stats_by_tablet_id, tablets[0]->id());
+    ASSERT_NE(nullptr, stats);
+    ASSERT_TRUE((*stats)->has_on_disk_size());
+    ASSERT_TRUE((*stats)->has_live_row_count());
+    ASSERT_EQ(1024, (*stats)->on_disk_size());
+    ASSERT_EQ(10, (*stats)->live_row_count());
+  }
+  {
+    const auto* stats = FindOrNull(stats_by_tablet_id, tablets[1]->id());
+    ASSERT_NE(nullptr, stats);
+    ASSERT_TRUE((*stats)->has_on_disk_size());
+    ASSERT_FALSE((*stats)->has_live_row_count());
+    ASSERT_EQ(2048, (*stats)->on_disk_size());
+  }
+  {
+    const auto* stats = FindOrNull(stats_by_tablet_id, tablets[2]->id());
+    ASSERT_NE(nullptr, stats);
+    ASSERT_FALSE((*stats)->has_on_disk_size());
+    ASSERT_FALSE((*stats)->has_live_row_count());
+  }
+}
+
 TEST_F(MasterTest, TestDeletedTablesAndTabletsCleanup) {
   FLAGS_enable_metadata_cleanup_for_deleted_tables_and_tablets = true;
   FLAGS_metadata_for_deleted_table_and_tablet_reserved_secs = 1;
diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto
index 12b74f4d6..fc18bd54d 100644
--- a/src/kudu/master/master.proto
+++ b/src/kudu/master/master.proto
@@ -690,6 +690,19 @@ message ListTablesResponsePB {
 
 message GetTableStatisticsRequestPB {
   required TableIdentifierPB table = 1;
+
+  // Whether to include per-tablet statistics in the response. Disabled by
+  // default to avoid the cost of collecting them when they are not needed.
+  optional bool include_tablet_statistics = 2;
+}
+
+message TabletStatisticsPB {
+  optional bytes tablet_id = 1;
+
+  // The on-disk size and live row count come from the latest leader tablet
+  // report. They are left unset if no leader has reported for the tablet yet.
+  optional int64 on_disk_size = 2;
+  optional int64 live_row_count = 3;
 }
 
 message GetTableStatisticsResponsePB {
@@ -703,6 +716,9 @@ message GetTableStatisticsResponsePB {
   // The table limit
   optional int64 disk_size_limit = 4;
   optional int64 row_count_limit = 5;
+
+  // Per-tablet statistics.
+  repeated TabletStatisticsPB tablet_statistics = 6;
 }
 
 // This data structure is used to specify a table's partition key.
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index ef00bd516..ec9961eb6 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -10036,6 +10036,25 @@ TEST_F(ToolTest, TestGetTableStatisticsNotSupported) {
   ASSERT_STR_CONTAINS(stdout, "live row count: N/A");
 }
 
+TEST_F(ToolTest, TestGetTableStatisticsShowTablets) {
+  NO_FATALS(StartExternalMiniCluster());
+
+  TestWorkload workload(cluster_.get());
+  workload.set_num_replicas(1);
+  workload.Setup();
+
+  string stdout;
+  NO_FATALS(RunActionStdoutString(
+      Substitute("table statistics $0 $1 --show_tablets",
+                 cluster_->master()->bound_rpc_addr().ToString(),
+                 TestWorkload::kDefaultTableName),
+      &stdout));
+  ASSERT_STR_CONTAINS(stdout, string("TABLE ") + TestWorkload::kDefaultTableName);
+  ASSERT_STR_CONTAINS(stdout, "tablet_id");
+  ASSERT_STR_CONTAINS(stdout, "on_disk_size");
+  ASSERT_STR_CONTAINS(stdout, "live_row_count");
+}
+
 // Run one of the cluster-targeted subcommands verifying the functionality of
 // the --connection_negotiation_timeout command-line option (the RPC
 // connection negotiation timeout).
diff --git a/src/kudu/tools/tool_action_common.cc b/src/kudu/tools/tool_action_common.cc
index 97e2ea335..0a0881a7b 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ b/src/kudu/tools/tool_action_common.cc
@@ -1342,5 +1342,17 @@ Status LeaderMasterProxy::SyncRpc(
                              const ResponseCallback&)>& func,
     std::vector<uint32_t> required_feature_flags);
 
+template
+Status LeaderMasterProxy::SyncRpc(
+    const master::GetTableStatisticsRequestPB& req,
+    master::GetTableStatisticsResponsePB* resp,
+    string func_name,
+    const std::function<void(MasterServiceProxy*,
+                             const master::GetTableStatisticsRequestPB&,
+                             master::GetTableStatisticsResponsePB*,
+                             RpcController*,
+                             const ResponseCallback&)>& func,
+    std::vector<uint32_t> required_feature_flags);
+
 } // namespace tools
 } // namespace kudu
diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc
index a6bab8cfb..868cfd0cc 100644
--- a/src/kudu/tools/tool_action_table.cc
+++ b/src/kudu/tools/tool_action_table.cc
@@ -52,6 +52,7 @@
 #include "kudu/common/wire_protocol.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/stl_util.h"
+#include "kudu/gutil/strings/human_readable.h"
 #include "kudu/gutil/strings/join.h"
 #include "kudu/gutil/strings/numbers.h"
 #include "kudu/gutil/strings/split.h"
@@ -91,6 +92,8 @@ using kudu::client::KuduTableStatistics;
 using kudu::client::KuduValue;
 using kudu::client::internal::ReplicaController;
 using kudu::iequals;
+using kudu::master::GetTableStatisticsRequestPB;
+using kudu::master::GetTableStatisticsResponsePB;
 using kudu::master::ListInFlightTablesRequestPB;
 using kudu::master::ListInFlightTablesResponsePB;
 using kudu::master::MasterServiceProxy;
@@ -127,6 +130,9 @@ DEFINE_bool(list_tablets, false,
 DEFINE_bool(show_table_info, false,
             "Include extra information such as number of tablets, replicas, "
             "and live row count for a table in the output");
+DEFINE_bool(show_tablets, false,
+            "Also list per-tablet on-disk size and live row count under the "
+            "table-level statistics output.");
 DEFINE_bool(show_tablet_partition_info, false,
             "Include partition keys information corresponding to tablet in the output.");
 
@@ -1598,13 +1604,55 @@ Status GetTableStatistics(const RunnerContext& context) {
   client::sp::shared_ptr<KuduClient> client;
   RETURN_NOT_OK(CreateKuduClient(context, &client));
 
-  unique_ptr<KuduTableStatistics> statistics;
-  KuduTableStatistics *table_statistics;
-  RETURN_NOT_OK_PREPEND(client->GetTableStatistics(table_name, &table_statistics),
-                        "failed to get table statistics.");
-  statistics.reset(table_statistics);
+  // Use the client API when per-tablet statistics aren't requested.
+  if (!FLAGS_show_tablets) {
+    unique_ptr<KuduTableStatistics> statistics;
+    KuduTableStatistics* table_statistics;
+    RETURN_NOT_OK_PREPEND(client->GetTableStatistics(table_name, &table_statistics),
+                          "failed to get table statistics.");
+    statistics.reset(table_statistics);
+    cout << "TABLE " << table_name << endl;
+    cout << statistics->ToString() << endl;
+    return Status::OK();
+  }
+
+  // Otherwise a single RPC returns both the table-level and per-tablet
+  // statistics, avoiding a redundant round-trip.
+  LeaderMasterProxy proxy;
+  RETURN_NOT_OK(proxy.Init(context));
+
+  GetTableStatisticsRequestPB req;
+  GetTableStatisticsResponsePB resp;
+  req.mutable_table()->set_table_name(table_name);
+  req.set_include_tablet_statistics(true);
+  RETURN_NOT_OK((proxy.SyncRpc<GetTableStatisticsRequestPB, GetTableStatisticsResponsePB>(
+      req, &resp, "GetTableStatistics", &MasterServiceProxy::GetTableStatisticsAsync)));
+
+  if (resp.has_error()) {
+    return StatusFromPB(resp.error().status());
+  }
+
   cout << "TABLE " << table_name << endl;
-  cout << statistics->ToString() << endl;
+  cout << Substitute("on disk size: $0\n"
+                     "live row count: $1\n"
+                     "on disk size limit: $2\n"
+                     "live row count limit: $3\n",
+                     resp.has_on_disk_size() ? std::to_string(resp.on_disk_size()) : "N/A",
+                     resp.has_live_row_count() ? std::to_string(resp.live_row_count()) : "N/A",
+                     resp.has_disk_size_limit() ? std::to_string(resp.disk_size_limit()) : "N/A",
+                     resp.has_row_count_limit() ? std::to_string(resp.row_count_limit()) : "N/A")
+       << endl;
+
+  cout << "TABLETS" << endl;
+  DataTable data_table({ "tablet_id", "on_disk_size", "live_row_count" });
+  for (const auto& t : resp.tablet_statistics()) {
+    data_table.AddRow({
+        t.tablet_id(),
+        t.has_on_disk_size() ? HumanReadableNumBytes::ToString(t.on_disk_size()) : "N/A",
+        t.has_live_row_count() ? std::to_string(t.live_row_count()) : "N/A",
+    });
+  }
+  RETURN_NOT_OK(data_table.PrintTo(cout));
 
   return Status::OK();
 }
@@ -2238,6 +2286,7 @@ unique_ptr<Mode> BuildTableMode() {
       ClusterActionBuilder("statistics", &GetTableStatistics)
       .Description("Get table statistics")
       .AddRequiredParameter({ kTableNameArg, "Name of the table to get statistics" })
+      .AddOptionalParameter("show_tablets")
       .Build();
 
   unique_ptr<Action> create_table =