You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2018/11/20 17:05:49 UTC

[1/3] kudu git commit: [tools] ksck: Add information about replica counts to plain ksck output

Repository: kudu
Updated Branches:
  refs/heads/master 0f3696448 -> e2cfbbf6b


[tools] ksck: Add information about replica counts to plain ksck output

This adds some information about replica counts on tablet servers to the
output of ksck when ksck is in PLAIN_* mode (i.e. not JSON output). It
outputs a 5-number summary of the distribution of replicas and lists any
outliers:

Tablet Replica Count Summary
   Statistic    | Replica Count
----------------+---------------
 Minimum        | 1646
 First Quartile | 3672
 Median         | 4075
 Third Quartile | 4242
 Maximum        | 4600

Tablet Replica Count Outliers
 Type  |               UUID               |              Host              | Replica Count
-------+----------------------------------+--------------------------------+---------------
 Small | cc32936bc8594948a04fd4240da36aed | vc1304.halxg.cloudera.com:7050 | 1646

In PLAIN_FULL mode it additionally outputs the replica count for every
tablet server:

Tablet Replica Count by Tablet Server
               UUID               |              Host              | Replica Count
----------------------------------+--------------------------------+---------------
 09d6bf7a02124145b43f43cb7a667b3d | vc1314.halxg.cloudera.com:7050 | 100
 23d473f441674d43807fd9e631862bfd | vc1308.halxg.cloudera.com:7050 | 100
 2fb5cdac22b0418bb2df456906e42eb4 | vc1306.halxg.cloudera.com:7050 | 101
 70f7ee61ead54b1885d819f354eb3405 | vc1316.halxg.cloudera.com:7050 | 95
 72fcec63e96f4248ae39d114eb3cd7c9 | vc1318.halxg.cloudera.com:7050 | 94
 86708813b37a44bd8e92c711211c8685 | vc1310.halxg.cloudera.com:7050 | 96
 a662440710624c02bd5612df32cb0235 | vc1302.halxg.cloudera.com:7050 | 101
 c9633273962a4521a32d5e177a118a84 | vc1312.halxg.cloudera.com:7050 | 101
 cc32936bc8594948a04fd4240da36aed | vc1304.halxg.cloudera.com:7050 | 76

I also tested it against an empty cluster.

There's no unit tests added, just because our current testing setup for
ksck makes it really painful to add one for this, and it seemed easy
enough to check out manually. Probably, a follow up should straighten
out ksck-test to make testing ksck changes easier.

Change-Id: I7e5373033ab84c1e34f9519eb9bd4e04a652c595
Reviewed-on: http://gerrit.cloudera.org:8080/11958
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b8e64989
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b8e64989
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b8e64989

Branch: refs/heads/master
Commit: b8e6498987731cfa4cbabcf817e1de600f96101f
Parents: 0f36964
Author: Will Berkeley <wd...@gmail.org>
Authored: Mon Nov 19 15:44:59 2018 -0800
Committer: Will Berkeley <wd...@gmail.com>
Committed: Tue Nov 20 10:12:43 2018 +0000

----------------------------------------------------------------------
 src/kudu/tools/ksck_results.cc | 123 ++++++++++++++++++++++++++++++++++++
 src/kudu/tools/ksck_results.h  |   8 +++
 2 files changed, 131 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/b8e64989/src/kudu/tools/ksck_results.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_results.cc b/src/kudu/tools/ksck_results.cc
index 8fd5601..fc78262 100644
--- a/src/kudu/tools/ksck_results.cc
+++ b/src/kudu/tools/ksck_results.cc
@@ -275,6 +275,10 @@ Status KsckResults::PrintTo(PrintMode mode, ostream& out) {
     out << endl;
   }
 
+  // Add in a summary of the tablet replicas by tablet server, so it's easy to
+  // see if some tablet servers have too many replicas.
+  RETURN_NOT_OK(PrintReplicaCountByTserverSummary(mode, *this, out));
+
   // Next, report on checksum scans.
   RETURN_NOT_OK(PrintChecksumResults(checksum_results, out));
   if (!checksum_results.tables.empty()) {
@@ -524,6 +528,125 @@ Status PrintTabletSummaries(const vector<KsckTabletSummary>& tablet_summaries,
   return Status::OK();
 }
 
+Status PrintReplicaCountByTserverSummary(PrintMode mode,
+                                         const KsckResults& results,
+                                         std::ostream& out) {
+  DCHECK(mode == PrintMode::PLAIN_FULL || mode == PrintMode::PLAIN_CONCISE);
+
+  // Collate the counts by tablet servers. We populate the map with the UUIDs
+  // from 'tserver_summaries' so we don't miss empty tablet servers.
+  map<string, std::pair<string, int>> host_and_replica_count_by_tserver;
+  for (const auto& tserver : results.tserver_summaries) {
+    EmplaceIfNotPresent(&host_and_replica_count_by_tserver,
+                        tserver.uuid,
+                        std::make_pair(tserver.address, 0));
+  }
+  for (const auto& tablet : results.tablet_summaries) {
+    for (const auto& replica : tablet.replicas) {
+      const string address = replica.ts_address ? *replica.ts_address :
+                                                  "unavailable";
+      LookupOrEmplace(&host_and_replica_count_by_tserver,
+                      replica.ts_uuid,
+                      std::make_pair(address, 0)).second++;
+    }
+  }
+
+  if (host_and_replica_count_by_tserver.empty()) {
+    return Status::OK();
+  }
+
+  // Compute a 5-number summary and print that plus outliers.
+  typedef std::tuple<string, string, int> UuidHostCount;
+  vector<UuidHostCount> tservers_sorted_by_replica_count;
+  tservers_sorted_by_replica_count.reserve(host_and_replica_count_by_tserver.size());
+  for (const auto& entry : host_and_replica_count_by_tserver) {
+    tservers_sorted_by_replica_count.emplace_back(entry.first,
+                                                  entry.second.first,
+                                                  entry.second.second);
+  }
+
+  DataTable table({ "Statistic", "Replica Count" });
+  std::sort(tservers_sorted_by_replica_count.begin(),
+            tservers_sorted_by_replica_count.end(),
+            [](const UuidHostCount& left, const UuidHostCount& right) {
+              return std::get<2>(left) < std::get<2>(right);
+            });
+  const auto num_tservers = tservers_sorted_by_replica_count.size();
+  const auto min = std::get<2>(tservers_sorted_by_replica_count[0]);
+  const auto first_quartile =
+      std::get<2>(tservers_sorted_by_replica_count[num_tservers / 4]);
+  const auto median = std::get<2>(tservers_sorted_by_replica_count[num_tservers / 2]);
+  const auto third_quartile =
+      std::get<2>(tservers_sorted_by_replica_count[3 * num_tservers / 4]);
+  const auto max = std::get<2>(tservers_sorted_by_replica_count[num_tservers - 1]);
+  table.AddRow({ "Minimum", std::to_string(min) });
+  table.AddRow({ "First Quartile", std::to_string(first_quartile) });
+  table.AddRow({ "Median", std::to_string(median) });
+  table.AddRow({ "Third Quartile", std::to_string(third_quartile) });
+  table.AddRow({ "Maximum", std::to_string(max) });
+
+  out << "Tablet Replica Count Summary" << endl;
+  RETURN_NOT_OK(table.PrintTo(out));
+  out << endl;
+
+  // Now outliers, if there are any. We use a standard definition: an outlier
+  // is a value more than 1.5 * (third - first quartile) below the first or
+  // above the third quartile.
+  const auto interquartile_range = third_quartile - first_quartile;
+  const auto below_is_outlier = first_quartile - 1.5 * interquartile_range;
+  const auto above_is_outlier = third_quartile + 1.5 * interquartile_range;
+  auto small_outliers = std::partition_point(
+      tservers_sorted_by_replica_count.rbegin(),
+      tservers_sorted_by_replica_count.rend(),
+      [&](const UuidHostCount val) {
+        return std::get<2>(val) >= below_is_outlier;
+      });
+  auto big_outliers = std::partition_point(
+      tservers_sorted_by_replica_count.begin(),
+      tservers_sorted_by_replica_count.end(),
+      [&](const UuidHostCount val) {
+        return std::get<2>(val) <= above_is_outlier;
+      });
+
+  int num_outliers = 0;
+  DataTable outliers({ "Type", "UUID", "Host", "Replica Count" });
+  for (auto it = big_outliers; it != tservers_sorted_by_replica_count.end(); ++it) {
+    num_outliers++;
+    const auto& uuid = std::get<0>(*it);
+    const auto& host = std::get<1>(*it);
+    const auto& count = std::get<2>(*it);
+    outliers.AddRow({ "Big", uuid, host, std::to_string(count) });
+  }
+  for (auto it = small_outliers; it != tservers_sorted_by_replica_count.rend(); ++it) {
+    num_outliers++;
+    const auto& uuid = std::get<0>(*it);
+    const auto& host = std::get<1>(*it);
+    const auto& count = std::get<2>(*it);
+    outliers.AddRow({ "Small", uuid, host, std::to_string(count) });
+  }
+
+  if (num_outliers > 0) {
+    out << "Tablet Replica Count Outliers" << endl;
+    RETURN_NOT_OK(outliers.PrintTo(out));
+    out << endl;
+  }
+
+  // If we're in verbose mode, also dump all the info.
+  if (mode == PrintMode::PLAIN_FULL) {
+    out << "Tablet Replica Count by Tablet Server" << endl;
+    DataTable table({ "UUID", "Host", "Replica Count" });
+    for (const auto& entry : host_and_replica_count_by_tserver) {
+      table.AddRow({ entry.first,
+                     entry.second.first,
+                     std::to_string(entry.second.second) });
+    }
+    RETURN_NOT_OK(table.PrintTo(out));
+    out << endl;
+  }
+
+  return Status::OK();
+}
+
 Status PrintChecksumResults(const KsckChecksumResults& checksum_results,
                             std::ostream& out) {
   if (checksum_results.tables.empty()) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/b8e64989/src/kudu/tools/ksck_results.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck_results.h b/src/kudu/tools/ksck_results.h
index 21a3112..4b8a596 100644
--- a/src/kudu/tools/ksck_results.h
+++ b/src/kudu/tools/ksck_results.h
@@ -353,6 +353,14 @@ Status PrintConsensusMatrix(const std::vector<std::string>& server_uuids,
                             const KsckConsensusStateMap& consensus_states,
                             std::ostream& out);
 
+// Print to 'out' a table summarizing the counts of tablet replicas in the
+// cluster. 'mode' must be PLAIN_FULL or PLAIN_CONCISE. In PLAIN_CONCISE mode,
+// summary information will be printed, while in PLAIN_FULL mode, the counts for
+// every tablet server will be printed as well.
+Status PrintReplicaCountByTserverSummary(PrintMode mode,
+                                         const KsckResults& results,
+                                         std::ostream& out);
+
 Status PrintChecksumResults(const KsckChecksumResults& checksum_results,
                             std::ostream& out);
 


[2/3] kudu git commit: [tools] Don't count tombstoned replicas in `kudu remote_replica check`

Posted by al...@apache.org.
[tools] Don't count tombstoned replicas in `kudu remote_replica check`

Counting them as not running made the tool pretty useless on any
realistic cluster, where it's expected to have some tombstones. For
example, it wasn't possible to use this tool to tell when all the tablet
replicas on a tablet server had finished bootstrapping.

I tested this against a server that previously failed the check because
it had about 6000/10000 replicas tombstoned in STOPPED and INITIALIZED
states. It now passes the check.

Change-Id: Ibbdde768cce5f1d27064b7276944a585b41a2f03
Reviewed-on: http://gerrit.cloudera.org:8080/11962
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Kudu Jenkins
Reviewed-by: Attila Bukor <ab...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/77c729d5
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/77c729d5
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/77c729d5

Branch: refs/heads/master
Commit: 77c729d58360d56de7b2bdab9304b867faee02a8
Parents: b8e6498
Author: Will Berkeley <wd...@gmail.org>
Authored: Tue Nov 20 00:37:00 2018 -0800
Committer: Will Berkeley <wd...@gmail.com>
Committed: Tue Nov 20 10:12:53 2018 +0000

----------------------------------------------------------------------
 src/kudu/tablet/metadata.proto               |  2 +-
 src/kudu/tools/tool_action_remote_replica.cc | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/77c729d5/src/kudu/tablet/metadata.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/metadata.proto b/src/kudu/tablet/metadata.proto
index 0c3110d..8905249 100644
--- a/src/kudu/tablet/metadata.proto
+++ b/src/kudu/tablet/metadata.proto
@@ -165,7 +165,7 @@ enum TabletStatePB {
   // state Peers are available for client RPCs.
   RUNNING = 1;
 
-  // The tablet failed to for some reason. TabletReplica::error() will return
+  // The tablet failed for some reason. TabletReplica::error() will return
   // the reason for the failure.
   FAILED = 2;
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/77c729d5/src/kudu/tools/tool_action_remote_replica.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_remote_replica.cc b/src/kudu/tools/tool_action_remote_replica.cc
index a62d695..9ffb195 100644
--- a/src/kudu/tools/tool_action_remote_replica.cc
+++ b/src/kudu/tools/tool_action_remote_replica.cc
@@ -200,7 +200,9 @@ Status CheckReplicas(const RunnerContext& context) {
   bool all_running = true;
   for (const auto& r : replicas) {
     const TabletStatusPB& rs = r.tablet_status();
-    if (rs.state() != tablet::RUNNING) {
+    // It's ok if the tablet replica isn't running if it's tombstoned.
+    if (rs.state() != tablet::RUNNING &&
+        rs.tablet_data_state() != tablet::TABLET_DATA_TOMBSTONED) {
       cerr << "Tablet id: " << rs.tablet_id() << " is "
            << tablet::TabletStatePB_Name(rs.state()) << endl;
       all_running = false;
@@ -208,10 +210,10 @@ Status CheckReplicas(const RunnerContext& context) {
   }
 
   if (all_running) {
-    cout << "All tablets are running" << endl;
+    cout << "All tablet replicas are running" << endl;
     return Status::OK();
   }
-  return Status::IllegalState("Not all tablets are running");
+  return Status::IllegalState("Not all tablet replicas are running");
 }
 
 Status DeleteReplica(const RunnerContext& context) {
@@ -394,7 +396,10 @@ Status UnsafeChangeConfig(const RunnerContext& context) {
 unique_ptr<Mode> BuildRemoteReplicaMode() {
   unique_ptr<Action> check_replicas =
       ActionBuilder("check", &CheckReplicas)
-      .Description("Check if all tablet replicas on a Kudu Tablet Server are running")
+      .Description("Check if all tablet replicas on a Kudu tablet server are "
+                   "running. Tombstoned replica do not count as not running, "
+                   "because they are just records of the previous existence of "
+                   "a replica.")
       .AddRequiredParameter({ kTServerAddressArg, kTServerAddressDesc })
       .Build();
 


[3/3] kudu git commit: [util] more information on fault injection

Posted by al...@apache.org.
[util] more information on fault injection

Log more information on injected fault: name of the process and PID.
That helps to parse concatenated logs: e.g., that useful in parsing
logs from scenarios run via dist-test.

Change-Id: Id0f094006a89fd271091d1bde13d301ea359ae61
Reviewed-on: http://gerrit.cloudera.org:8080/11960
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley <wd...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e2cfbbf6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e2cfbbf6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e2cfbbf6

Branch: refs/heads/master
Commit: e2cfbbf6bc56f1dcf6d7402b813f41c09711a3e8
Parents: 77c729d
Author: Alexey Serbin <al...@apache.org>
Authored: Mon Nov 19 21:00:45 2018 -0800
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Tue Nov 20 16:59:58 2018 +0000

----------------------------------------------------------------------
 src/kudu/util/fault_injection.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/e2cfbbf6/src/kudu/util/fault_injection.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/fault_injection.cc b/src/kudu/util/fault_injection.cc
index 6638bb6..2432872 100644
--- a/src/kudu/util/fault_injection.cc
+++ b/src/kudu/util/fault_injection.cc
@@ -20,11 +20,14 @@
 #include <unistd.h>
 
 #include <ostream>
+#include <string>
 
+#include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include "kudu/gutil/dynamic_annotations.h"
 #include "kudu/gutil/once.h"
+#include "kudu/gutil/strings/substitute.h"
 #include "kudu/util/debug/leakcheck_disabler.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/random.h"
@@ -54,7 +57,9 @@ void DoMaybeFault(const char* fault_str, double fraction) {
   if (PREDICT_TRUE(g_random->NextDoubleFraction() >= fraction)) {
     return;
   }
-  LOG(ERROR) << "Injecting fault: " << fault_str << " (process will exit)";
+  LOG(ERROR) << strings::Substitute(
+      "injecting fault for $0 (pid $1): $2 (process will exit)",
+      google::ProgramInvocationShortName(), getpid(), fault_str);
   // _exit will exit the program without running atexit handlers. This more
   // accurately simulates a crash.
   _exit(kExitStatus);