You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2019/06/02 02:17:55 UTC
[kudu] branch master updated: [hms] Add a tool to list all Kudu HMS
entries
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 11a29a9 [hms] Add a tool to list all Kudu HMS entries
11a29a9 is described below
commit 11a29a9e8d6c095a0200dfabe0afb6d203559342
Author: Grant Henke <gr...@apache.org>
AuthorDate: Fri May 31 18:45:47 2019 -0500
[hms] Add a tool to list all Kudu HMS entries
Adds a tool to list the Kudu HMS entries with optional
columns in various formats.
Change-Id: Ib18cde6e92e799390bb977562fd8d2eac2dae026
Reviewed-on: http://gerrit.cloudera.org:8080/13490
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
src/kudu/tools/kudu-tool-test.cc | 75 +++++++++++++++++++++++++++++++++++
src/kudu/tools/tool_action_hms.cc | 83 +++++++++++++++++++++++++++++++++++++--
2 files changed, 155 insertions(+), 3 deletions(-)
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index d872724..727046a 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -950,6 +950,7 @@ TEST_F(ToolTest, TestTopLevelHelp) {
"cluster.*Kudu cluster",
"diagnose.*Diagnostic tools.*",
"fs.*Kudu filesystem",
+ "hms.*Hive Metastores",
"local_replica.*tablet replicas",
"master.*Kudu Master",
"pbc.*protobuf container",
@@ -970,6 +971,18 @@ TEST_F(ToolTest, TestTopLevelHelp) {
TEST_F(ToolTest, TestModeHelp) {
{
const vector<string> kFsModeRegexes = {
+ "check.*Check metadata consistency",
+ "downgrade.*Downgrade the metadata",
+ "fix.*Fix automatically-repairable metadata",
+ "list.*List the Kudu table HMS entries",
+ "precheck.*Check that the Kudu cluster is prepared",
+ };
+ NO_FATALS(RunTestHelp("hms", kFsModeRegexes));
+ NO_FATALS(RunTestHelp("hms not_a_mode", kFsModeRegexes,
+ Status::InvalidArgument("unknown command 'not_a_mode'")));
+ }
+ {
+ const vector<string> kFsModeRegexes = {
"check.*Kudu filesystem for inconsistencies",
"dump.*Dump a Kudu filesystem",
"format.*new Kudu filesystem",
@@ -3949,6 +3962,68 @@ TEST_F(ToolTest, TestHmsPrecheck) {
}), tables);
}
+TEST_F(ToolTest, TestHmsList) {
+ ExternalMiniClusterOptions opts;
+ opts.hms_mode = HmsMode::ENABLE_HIVE_METASTORE;
+ opts.enable_kerberos = EnableKerberos();
+ NO_FATALS(StartExternalMiniCluster(std::move(opts)));
+
+ string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+ thrift::ClientOptions hms_opts;
+ hms_opts.enable_kerberos = EnableKerberos();
+ hms_opts.service_principal = "hive";
+ HmsClient hms_client(cluster_->hms()->address(), hms_opts);
+ ASSERT_OK(hms_client.Start());
+ ASSERT_TRUE(hms_client.IsConnected());
+
+ FLAGS_hive_metastore_uris = cluster_->hms()->uris();
+ FLAGS_hive_metastore_sasl_enabled = EnableKerberos();
+ HmsCatalog hms_catalog(master_addr);
+ ASSERT_OK(hms_catalog.Start());
+
+ shared_ptr<KuduClient> kudu_client;
+ ASSERT_OK(cluster_->CreateClient(nullptr, &kudu_client));
+
+ // Create a simple Kudu table so we can use the schema.
+ shared_ptr<KuduTable> simple_table;
+ ASSERT_OK(CreateKuduTable(kudu_client, "default.simple"));
+ ASSERT_OK(kudu_client->OpenTable("default.simple", &simple_table));
+
+ string kUsername = "alice";
+ ASSERT_OK(hms_catalog.CreateTable(
+ "1", "default.table1", kUsername, KuduSchema::ToSchema(simple_table->schema()),
+ hms::HmsClient::kManagedTable));
+ ASSERT_OK(hms_catalog.CreateTable(
+ "2", "default.table2", boost::none, KuduSchema::ToSchema(simple_table->schema()),
+ hms::HmsClient::kExternalTable));
+
+ // Test the output when HMS integration is disabled.
+ string err;
+ RunActionStderrString(Substitute("hms list $0", master_addr), &err);
+ ASSERT_STR_CONTAINS(err,
+ "Configuration error: the Kudu leader master is not configured with "
+ "the Hive Metastore integration");
+
+ // Enable the HMS integration.
+ cluster_->ShutdownNodes(cluster::ClusterNodes::MASTERS_ONLY);
+ cluster_->EnableMetastoreIntegration();
+ ASSERT_OK(cluster_->Restart());
+
+ // Run the list tool with the defaults. atabase,table,type,$0
+ string out;
+ RunActionStdoutString(Substitute("hms list $0", master_addr), &out);
+ ASSERT_STR_CONTAINS(out,
+ "default | table1 | MANAGED_TABLE | default.table1");
+ ASSERT_STR_CONTAINS(out,
+ "default | table2 | EXTERNAL_TABLE | default.table2");
+
+ // Run the list tool filtering columns and using a different format.
+ RunActionStdoutString(Substitute("hms list --columns table,owner,kudu.table_id, --format=csv $0",
+ master_addr), &out);
+ ASSERT_STR_CONTAINS(out, "table1,alice,1");
+ ASSERT_STR_CONTAINS(out, "table2,,");
+}
+
// This test is parameterized on the serialization mode and Kerberos.
class ControlShellToolTest :
public ToolTest,
diff --git a/src/kudu/tools/tool_action_hms.cc b/src/kudu/tools/tool_action_hms.cc
index f96d462..dfd2f69 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -27,6 +27,7 @@
#include <utility>
#include <vector>
+#include <boost/algorithm/string/predicate.hpp>
#include <boost/optional/optional.hpp>
#include <gflags/gflags.h>
#include <gflags/gflags_declare.h>
@@ -38,6 +39,7 @@
#include "kudu/common/schema.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/strings/split.h"
+#include "kudu/gutil/strings/stringpiece.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/hms/hive_metastore_types.h"
#include "kudu/hms/hms_catalog.h"
@@ -48,6 +50,7 @@
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
+DECLARE_string(columns);
DECLARE_bool(force);
DECLARE_bool(hive_metastore_sasl_enabled);
DECLARE_int64(timeout_ms);
@@ -212,10 +215,10 @@ Status PrintTables(const string& master_addrs,
"HMS database",
"HMS table",
"HMS table type",
- Substitute("HMS $0", HmsClient::kStorageHandlerKey),
Substitute("HMS $0", HmsClient::kKuduTableNameKey),
Substitute("HMS $0", HmsClient::kKuduTableIdKey),
Substitute("HMS $0", HmsClient::kKuduMasterAddrsKey),
+ Substitute("HMS $0", HmsClient::kStorageHandlerKey),
});
for (auto& pair : tables) {
vector<string> row;
@@ -232,10 +235,10 @@ Status PrintTables(const string& master_addrs,
row.emplace_back(hms_table.dbName);
row.emplace_back(hms_table.tableName);
row.emplace_back(hms_table.tableType);
- row.emplace_back(hms_table.parameters[HmsClient::kStorageHandlerKey]);
row.emplace_back(hms_table.parameters[HmsClient::kKuduTableNameKey]);
row.emplace_back(hms_table.parameters[HmsClient::kKuduTableIdKey]);
row.emplace_back(hms_table.parameters[HmsClient::kKuduMasterAddrsKey]);
+ row.emplace_back(hms_table.parameters[HmsClient::kStorageHandlerKey]);
} else {
row.resize(10);
}
@@ -244,6 +247,51 @@ Status PrintTables(const string& master_addrs,
return table.PrintTo(out);
}
+// Prints catalog information about Kudu HMS tables in data table format to 'out'.
+Status PrintHMSTables(vector<hive::Table> tables, ostream& out) {
+ DataTable table({});
+ for (const auto& column : strings::Split(FLAGS_columns, ",", strings::SkipEmpty())) {
+ vector<string> values;
+ if (boost::iequals(column, "database")) {
+ for (auto& hms_table : tables) {
+ values.emplace_back(hms_table.dbName);
+ }
+ } else if (boost::iequals(column, "table")) {
+ for (auto& hms_table : tables) {
+ values.emplace_back(hms_table.tableName);
+ }
+ } else if (boost::iequals(column, "type")) {
+ for (auto& hms_table : tables) {
+ values.emplace_back(hms_table.tableType);
+ }
+ } else if (boost::iequals(column, "owner")) {
+ for (auto& hms_table : tables) {
+ values.emplace_back(hms_table.owner);
+ }
+ } else if (boost::iequals(column, HmsClient::kKuduTableNameKey)) {
+ for (auto& hms_table : tables) {
+ values.emplace_back(hms_table.parameters[HmsClient::kKuduTableNameKey]);
+ }
+ } else if (boost::iequals(column, HmsClient::kKuduTableIdKey)) {
+ for (auto& hms_table : tables) {
+ values.emplace_back(hms_table.parameters[HmsClient::kKuduTableIdKey]);
+ }
+ } else if (boost::iequals(column, HmsClient::kKuduMasterAddrsKey)) {
+ for (auto& hms_table : tables) {
+ values.emplace_back(hms_table.parameters[HmsClient::kKuduMasterAddrsKey]);
+ }
+ } else if (boost::iequals(column, HmsClient::kStorageHandlerKey)) {
+ for (auto& hms_table : tables) {
+ values.emplace_back(hms_table.parameters[HmsClient::kStorageHandlerKey]);
+ }
+ } else {
+ return Status::InvalidArgument("unknown column (--columns)", column);
+ }
+ table.AddColumn(column.ToString(), std::move(values));
+ }
+ return table.PrintTo(out);
+}
+
// A report of inconsistent tables in Kudu and the HMS catalogs.
struct CatalogReport {
// Kudu tables in the HMS catalog which have no corresponding table in the
@@ -717,6 +765,18 @@ Status FixHmsMetadata(const RunnerContext& context) {
return Status::RuntimeError("Failed to fix some catalog metadata inconsistencies");
}
+Status List(const RunnerContext& context) {
+ shared_ptr<KuduClient> kudu_client;
+ unique_ptr<HmsCatalog> hms_catalog;
+ string master_addrs;
+ RETURN_NOT_OK(Init(context, &kudu_client, &hms_catalog, &master_addrs));
+
+ vector<hive::Table> hms_tables;
+ RETURN_NOT_OK(hms_catalog->GetKuduTables(&hms_tables));
+
+ return PrintHMSTables(hms_tables, cout);
+}
+
Status Precheck(const RunnerContext& context) {
string master_addrs;
RETURN_NOT_OK(ParseMasterAddressesStr(context, &master_addrs));
@@ -771,7 +831,6 @@ Status Precheck(const RunnerContext& context) {
return Status::IllegalState("found tables in Kudu with case-conflicting names");
}
-// TODO(ghenke): Add dump tool that prints the Kudu HMS entries.
unique_ptr<Mode> BuildHmsMode() {
const string kHmsUrisDesc =
"Address of the Hive Metastore instance(s). If not set, the configuration "
@@ -820,6 +879,23 @@ unique_ptr<Mode> BuildHmsMode() {
.AddOptionalParameter("ignore_other_clusters")
.Build();
+ unique_ptr<Action> hms_list =
+ ActionBuilder("list", &List)
+ .Description("List the Kudu table HMS entries")
+ .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
+ .AddOptionalParameter("columns",
+ Substitute("database,table,type,$0",
+ HmsClient::kKuduTableNameKey),
+ Substitute("Comma-separated list of HMS entry fields to "
+ "include in output.\nPossible values: database, "
+ "table, type, owner, $0, $1, $2, $3",
+ HmsClient::kKuduTableNameKey,
+ HmsClient::kKuduTableIdKey,
+ HmsClient::kKuduMasterAddrsKey,
+ HmsClient::kStorageHandlerKey))
+ .AddOptionalParameter("format")
+ .Build();
+
unique_ptr<Action> hms_precheck =
ActionBuilder("precheck", &Precheck)
.Description("Check that the Kudu cluster is prepared to enable the Hive "
@@ -831,6 +907,7 @@ unique_ptr<Mode> BuildHmsMode() {
.AddAction(std::move(hms_check))
.AddAction(std::move(hms_downgrade))
.AddAction(std::move(hms_fix))
+ .AddAction(std::move(hms_list))
.AddAction(std::move(hms_precheck))
.Build();
}