You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/08/25 21:51:06 UTC
[5/5] kudu git commit: tool: add ksck
tool: add ksck
This patch adds ksck functionality to the new CLI tool. ksck is still
available as a library used by several integration tests.
Change-Id: Iad9aaffa5c0c77080dcaeb2cdfa572dcbeeb1407
Reviewed-on: http://gerrit.cloudera.org:8080/4121
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/86afadc7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/86afadc7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/86afadc7
Branch: refs/heads/master
Commit: 86afadc7fc6f9274402edcb3599c243ec2de0ccb
Parents: 7321b38
Author: Adar Dembo <ad...@cloudera.com>
Authored: Wed Aug 24 20:24:08 2016 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Aug 25 21:46:47 2016 +0000
----------------------------------------------------------------------
docs/release_notes.adoc | 3 +
src/kudu/tools/CMakeLists.txt | 7 +-
src/kudu/tools/kudu-ksck.cc | 151 -----------------------------
src/kudu/tools/kudu-tool-test.cc | 6 ++
src/kudu/tools/tool_action.h | 1 +
src/kudu/tools/tool_action_cluster.cc | 149 ++++++++++++++++++++++++++++
src/kudu/tools/tool_main.cc | 1 +
7 files changed, 162 insertions(+), 156 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/docs/release_notes.adoc
----------------------------------------------------------------------
diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc
index 38c3d7a..b629a2f 100644
--- a/docs/release_notes.adoc
+++ b/docs/release_notes.adoc
@@ -43,6 +43,9 @@ detailed below.
- The `kudu-pbc-dump` tool has been removed. The same functionality is now
implemented as `kudu pbc dump`.
+- The `kudu-ksck` tool has been removed. The same functionality is now
+ implemented as `kudu cluster ksck`.
+
- KuduSession methods in the C++ library are no longer advertised as thread-safe
to have one set of semantics for both C++ and Java Kudu client libraries.
http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/tools/CMakeLists.txt b/src/kudu/tools/CMakeLists.txt
index ce361f0..614bf11 100644
--- a/src/kudu/tools/CMakeLists.txt
+++ b/src/kudu/tools/CMakeLists.txt
@@ -82,13 +82,9 @@ target_link_libraries(ksck
${KUDU_BASE_LIBS}
)
-add_executable(kudu-ksck kudu-ksck.cc)
-target_link_libraries(kudu-ksck
- ksck
-)
-
add_executable(kudu
tool_action.cc
+ tool_action_cluster.cc
tool_action_fs.cc
tool_action_pbc.cc
tool_action_tablet.cc
@@ -100,6 +96,7 @@ target_link_libraries(kudu
kudu_common
kudu_fs
kudu_util
+ ksck
master
tserver
${KUDU_BASE_LIBS}
http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/kudu-ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-ksck.cc b/src/kudu/tools/kudu-ksck.cc
deleted file mode 100644
index 70a6ce1..0000000
--- a/src/kudu/tools/kudu-ksck.cc
+++ /dev/null
@@ -1,151 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-// Command line tool to run Ksck against a cluster. Defaults to running against a local Master
-// on the default RPC port. It verifies that all the reported Tablet Servers are running and that
-// the tablets are in a consistent state.
-
-#include <gflags/gflags.h>
-#include <glog/logging.h>
-#include <iostream>
-
-#include "kudu/gutil/strings/split.h"
-#include "kudu/gutil/strings/substitute.h"
-#include "kudu/tools/ksck_remote.h"
-#include "kudu/util/flags.h"
-#include "kudu/util/logging.h"
-
-#define PUSH_PREPEND_NOT_OK(s, statuses, msg) do { \
- ::kudu::Status _s = (s); \
- if (PREDICT_FALSE(!_s.ok())) { \
- statuses->push_back(string(msg) + ": " + _s.ToString()); \
- } \
-} while (0);
-
-using std::cerr;
-using std::cout;
-using std::endl;
-using std::shared_ptr;
-using std::vector;
-using strings::Substitute;
-
-DEFINE_string(master_address, "",
- "Address of master server to run against.");
-
-DEFINE_bool(checksum_scan, false,
- "Perform a checksum scan on data in the cluster.");
-
-DEFINE_string(tables, "",
- "Tables to check (comma-separated list of names). "
- "If not specified, checks all tables.");
-
-DEFINE_string(tablets, "",
- "Tablets to check (comma-separated list of IDs) "
- "If not specified, checks all tablets.");
-
-namespace kudu {
-namespace tools {
-
-static string GetKsckUsage(const char* progname) {
- string msg = Substitute("Usage: $0 --master_address=<addr> <flags>\n\n", progname);
- msg += "Check the health of a Kudu cluster.\n\n"
- "By default, ksck checks that master and tablet server processes are running,\n"
- "and that table metadata is consistent. Use the 'checksum' flag to check that\n"
- "tablet data is consistent (also see the 'tables' and 'tablets' flags below).\n"
- "Use the 'checksum_snapshot' along with 'checksum' if the table or tablets are\n"
- "actively receiving inserts or updates.";
- return msg;
-}
-
-// Run ksck.
-// Error information is appended to the provided vector.
-// If the vector is empty upon completion, ksck ran successfully.
-static void RunKsck(vector<string>* error_messages) {
- vector<Sockaddr> master_addrs;
- PUSH_PREPEND_NOT_OK(ParseAddressList(FLAGS_master_address,
- master::Master::kDefaultPort,
- &master_addrs),
- error_messages, "Unable to parse master address");
- // return early if the addresses could not be resolved
- if (!error_messages->empty()) return;
-
- shared_ptr<KsckMaster> master;
- PUSH_PREPEND_NOT_OK(RemoteKsckMaster::Build(master_addrs[0], &master),
- error_messages, "Unable to build KsckMaster");
- if (!error_messages->empty()) return;
- shared_ptr<KsckCluster> cluster(new KsckCluster(master));
- shared_ptr<Ksck> ksck(new Ksck(cluster));
-
- ksck->set_table_filters(strings::Split(FLAGS_tables, ",", strings::SkipEmpty()));
- ksck->set_tablet_id_filters(strings::Split(FLAGS_tablets, ",", strings::SkipEmpty()));
-
- // This is required for everything below.
- PUSH_PREPEND_NOT_OK(ksck->CheckMasterRunning(), error_messages,
- "Master aliveness check error");
- if (!error_messages->empty()) return;
-
- // This is also required for everything below.
- PUSH_PREPEND_NOT_OK(ksck->FetchTableAndTabletInfo(), error_messages,
- "Error fetching the cluster metadata from the Master server");
- if (!error_messages->empty()) return;
-
- PUSH_PREPEND_NOT_OK(ksck->FetchInfoFromTabletServers(), error_messages,
- "Error fetching info from tablet servers");
-
- // TODO: Add support for tables / tablets filter in the consistency check.
- PUSH_PREPEND_NOT_OK(ksck->CheckTablesConsistency(), error_messages,
- "Table consistency check error");
-
- if (FLAGS_checksum_scan) {
- PUSH_PREPEND_NOT_OK(ksck->ChecksumData(ChecksumOptions()),
- error_messages, "Checksum scan error");
- }
-}
-
-} // namespace tools
-} // namespace kudu
-
-int main(int argc, char** argv) {
- google::SetUsageMessage(kudu::tools::GetKsckUsage(argv[0]));
- if (argc < 2) {
- google::ShowUsageWithFlagsRestrict(argv[0], __FILE__);
- exit(1);
- }
- kudu::ParseCommandLineFlags(&argc, &argv, true);
- FLAGS_logtostderr = true;
- kudu::InitGoogleLoggingSafe(argv[0]);
-
- vector<string> error_messages;
- kudu::tools::RunKsck(&error_messages);
-
- // All good.
- if (error_messages.empty()) {
- cout << "OK" << endl;
- return 0;
- }
-
- // Something went wrong.
- cerr << "==================" << endl;
- cerr << "Errors:" << endl;
- cerr << "==================" << endl;
- for (const string& s : error_messages) {
- cerr << s << endl;
- }
- cerr << endl;
- cerr << "FAILED" << endl;
- return 1;
-}
http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/kudu-tool-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 2700fb1..ba748aa 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -137,6 +137,12 @@ TEST_F(ToolTest, TestModeHelp) {
};
NO_FATALS(RunTestHelp("tablet cmeta", kCmetaModeRegexes));
}
+ {
+ const vector<string> kClusterModeRegexes = {
+ "ksck.*Check the health of a Kudu cluster",
+ };
+ NO_FATALS(RunTestHelp("cluster", kClusterModeRegexes));
+ }
}
TEST_F(ToolTest, TestActionHelp) {
http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_action.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action.h b/src/kudu/tools/tool_action.h
index da5d1e9..7d79a4f 100644
--- a/src/kudu/tools/tool_action.h
+++ b/src/kudu/tools/tool_action.h
@@ -247,6 +247,7 @@ class Action {
};
// Returns new nodes for each major mode.
+std::unique_ptr<Mode> BuildClusterMode();
std::unique_ptr<Mode> BuildFsMode();
std::unique_ptr<Mode> BuildPbcMode();
std::unique_ptr<Mode> BuildTabletMode();
http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_action_cluster.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_cluster.cc b/src/kudu/tools/tool_action_cluster.cc
new file mode 100644
index 0000000..d198a9b
--- /dev/null
+++ b/src/kudu/tools/tool_action_cluster.cc
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/tools/tool_action.h"
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include <gflags/gflags.h>
+
+#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/strings/split.h"
+#include "kudu/master/master.h"
+#include "kudu/tools/ksck.h"
+#include "kudu/tools/ksck_remote.h"
+#include "kudu/util/net/net_util.h"
+#include "kudu/util/status.h"
+
+#define PUSH_PREPEND_NOT_OK(s, statuses, msg) do { \
+ ::kudu::Status _s = (s); \
+ if (PREDICT_FALSE(!_s.ok())) { \
+ statuses.push_back(string(msg) + ": " + _s.ToString()); \
+ } \
+} while (0);
+
+
+DEFINE_bool(checksum_scan, false,
+ "Perform a checksum scan on data in the cluster.");
+
+DEFINE_string(tables, "",
+ "Tables to check (comma-separated list of names). "
+ "If not specified, checks all tables.");
+
+DEFINE_string(tablets, "",
+ "Tablets to check (comma-separated list of IDs) "
+ "If not specified, checks all tablets.");
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::shared_ptr;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace kudu {
+namespace tools {
+
+namespace {
+
+Status RunKsck(const RunnerContext& context) {
+ vector<Sockaddr> master_addrs;
+ string master_address = FindOrDie(context.required_args, "master_address");
+ RETURN_NOT_OK_PREPEND(ParseAddressList(master_address,
+ master::Master::kDefaultPort,
+ &master_addrs),
+ "unable to parse master address");
+
+ shared_ptr<KsckMaster> master;
+ RETURN_NOT_OK_PREPEND(RemoteKsckMaster::Build(master_addrs[0], &master),
+ "unable to build KsckMaster");
+
+ shared_ptr<KsckCluster> cluster(new KsckCluster(master));
+ shared_ptr<Ksck> ksck(new Ksck(cluster));
+
+ ksck->set_table_filters(strings::Split(
+ FLAGS_tables, ",", strings::SkipEmpty()));
+ ksck->set_tablet_id_filters(strings::Split(
+ FLAGS_tablets, ",", strings::SkipEmpty()));
+
+ RETURN_NOT_OK_PREPEND(ksck->CheckMasterRunning(),
+ "master liveness check error");
+ RETURN_NOT_OK_PREPEND(ksck->FetchTableAndTabletInfo(),
+ "error fetching the cluster metadata from the Master server");
+
+ vector<string> error_messages;
+ PUSH_PREPEND_NOT_OK(ksck->FetchInfoFromTabletServers(), error_messages,
+ "error fetching info from tablet servers");
+
+ // TODO: Add support for tables / tablets filter in the consistency check.
+ PUSH_PREPEND_NOT_OK(ksck->CheckTablesConsistency(), error_messages,
+ "table consistency check error");
+
+ if (FLAGS_checksum_scan) {
+ PUSH_PREPEND_NOT_OK(ksck->ChecksumData(ChecksumOptions()),
+ error_messages, "checksum scan error");
+ }
+
+ // All good.
+ if (error_messages.empty()) {
+ cout << "OK" << endl;
+ return Status::OK();
+ }
+
+ // Something went wrong.
+ cerr << "==================" << endl;
+ cerr << "Errors:" << endl;
+ cerr << "==================" << endl;
+ for (const auto& s : error_messages) {
+ cerr << s << endl;
+ }
+ cerr << endl;
+ cerr << "FAILED" << endl;
+ return Status::RuntimeError("ksck discovered errors");
+}
+
+} // anonymous namespace
+
+unique_ptr<Mode> BuildClusterMode() {
+ // TODO: the long description looks pretty horrible, need to fix.
+ string desc = "Check the health of a Kudu cluster. By default, ksck checks "
+ "that master and tablet server processes are running, and that table "
+ "metadata is consistent. Use the 'checksum' flag to check that tablet "
+ "data is consistent (also see the 'tables' and 'tablets' flags). Use the "
+ "'checksum_snapshot' along with 'checksum' if the table or tablets are "
+ "actively receiving inserts or updates.";
+ unique_ptr<Action> ksck = ActionBuilder(
+ { "ksck", desc }, &RunKsck)
+ .AddRequiredParameter({ "master_address", "Kudu Master RPC address of form hostname:port" })
+ .AddOptionalParameter("checksum_scan")
+ .AddOptionalParameter("checksum_snapshot")
+ .AddOptionalParameter("tables")
+ .AddOptionalParameter("tablets")
+ .Build();
+
+ return ModeBuilder({ "cluster", "Operate on a Kudu cluster" })
+ .AddAction(std::move(ksck))
+ .Build();
+}
+
+} // namespace tools
+} // namespace kudu
+
http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_main.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_main.cc b/src/kudu/tools/tool_main.cc
index 18137a1..70fdfc0 100644
--- a/src/kudu/tools/tool_main.cc
+++ b/src/kudu/tools/tool_main.cc
@@ -111,6 +111,7 @@ int DispatchCommand(const vector<Mode*>& chain,
int RunTool(int argc, char** argv, bool show_help) {
unique_ptr<Mode> root =
ModeBuilder({ argv[0], "" }) // root mode description isn't printed
+ .AddMode(BuildClusterMode())
.AddMode(BuildFsMode())
.AddMode(BuildPbcMode())
.AddMode(BuildTabletMode())