You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2019/06/03 17:53:49 UTC
[kudu] 06/06: [hms] Support repairing entries with invalid IDs
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 1ccebcc43e28b86d58a6642d844958413a2f1354
Author: Grant Henke <gr...@apache.org>
AuthorDate: Sat Jun 1 10:51:15 2019 -0500
[hms] Support repairing entries with invalid IDs
This patch updates the hms tool to support fixing HMS
tables with missing or invalid kudu.table_id properties.
To support this, the patch changes the HMS plugin to
allow tables with unmatched IDs to be altered when
`kudu.check_id` is false in the environment context.
Change-Id: I437a8b42182dcfa09972f8059ccb6af280925085
Reviewed-on: http://gerrit.cloudera.org:8080/13492
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao <ha...@cloudera.com>
---
.../kudu/hive/metastore/KuduMetastorePlugin.java | 39 +++++++++++++++++++---
.../hive/metastore/TestKuduMetastorePlugin.java | 10 ++++++
src/kudu/hms/hms_catalog.cc | 14 +++++---
src/kudu/hms/hms_catalog.h | 9 ++---
src/kudu/hms/hms_client.cc | 1 +
src/kudu/hms/hms_client.h | 1 +
src/kudu/tools/kudu-tool-test.cc | 12 +++++++
src/kudu/tools/tool_action_hms.cc | 16 ++++++---
8 files changed, 84 insertions(+), 18 deletions(-)
diff --git a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
index c36bc8e..cb6e5b4 100644
--- a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
+++ b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
@@ -78,6 +78,8 @@ public class KuduMetastorePlugin extends MetaStoreEventListener {
static final String KUDU_MASTER_ADDRS_KEY = "kudu.master_addresses";
@VisibleForTesting
static final String KUDU_MASTER_EVENT = "kudu.master_event";
+ @VisibleForTesting
+ static final String KUDU_CHECK_ID_KEY = "kudu.check_id";
// System env to track if the HMS plugin validation should be skipped.
static final String SKIP_VALIDATION_ENV = "KUDU_SKIP_HMS_PLUGIN_VALIDATION";
@@ -208,10 +210,13 @@ public class KuduMetastorePlugin extends MetaStoreEventListener {
checkKuduProperties(newTable);
checkOnlyKuduMasterCanAlterSchema(tableEvent, oldTable, newTable);
// Check that the Kudu table ID isn't changing.
- String oldTableId = oldTable.getParameters().get(KUDU_TABLE_ID_KEY);
- String newTableId = newTable.getParameters().get(KUDU_TABLE_ID_KEY);
- if (!newTableId.equals(oldTableId)) {
- throw new MetaException("Kudu table ID does not match the existing HMS entry");
+
+ if (checkTableID(tableEvent)) {
+ String oldTableId = oldTable.getParameters().get(KUDU_TABLE_ID_KEY);
+ String newTableId = newTable.getParameters().get(KUDU_TABLE_ID_KEY);
+ if (!newTableId.equals(oldTableId)) {
+ throw new MetaException("Kudu table ID does not match the existing HMS entry");
+ }
}
} else {
// Allow non-Kudu tables to be altered without introducing Kudu-specific
@@ -312,10 +317,36 @@ public class KuduMetastorePlugin extends MetaStoreEventListener {
return false;
}
+ if (!properties.containsKey(KUDU_MASTER_EVENT)) {
+ return false;
+ }
+
return Boolean.parseBoolean(properties.get(KUDU_MASTER_EVENT));
}
/**
+ * Returns true if the table ID should be verified on an event.
+ * Defaults to true.
+ */
+ private boolean checkTableID(ListenerEvent event) {
+ EnvironmentContext environmentContext = event.getEnvironmentContext();
+ if (environmentContext == null) {
+ return true;
+ }
+
+ Map<String, String> properties = environmentContext.getProperties();
+ if (properties == null) {
+ return true;
+ }
+
+ if (!properties.containsKey(KUDU_CHECK_ID_KEY)) {
+ return true;
+ }
+
+ return Boolean.parseBoolean(properties.get(KUDU_CHECK_ID_KEY));
+ }
+
+ /**
* Returns true if the system env is set to skip validation.
*/
private static boolean skipsValidation() {
diff --git a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
index d0d3641..762d411 100644
--- a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
+++ b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
@@ -236,6 +236,16 @@ public class TestKuduMetastorePlugin {
assertTrue(e.getMessage().contains("Kudu table ID does not match the existing HMS entry"));
}
+ // Check that altering the Kudu table with a different table ID while
+ // setting kudu.check_id to false succeeds.
+ EnvironmentContext noCheckIdCtx = new EnvironmentContext(
+ ImmutableMap.of(KuduMetastorePlugin.KUDU_CHECK_ID_KEY, "false"));
+ client.alter_table_with_environmentContext(table.getDbName(), table.getTableName(),
+ newTable(table.getTableName()), noCheckIdCtx);
+ // Alter back for more testing below.
+ client.alter_table_with_environmentContext(table.getDbName(), table.getTableName(), table,
+ noCheckIdCtx);
+
// Try to alter the Kudu table with no storage handler.
try {
Table alteredTable = table.deepCopy();
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index c46a500..7c2219a 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -240,7 +240,8 @@ Status HmsCatalog::GetKuduTables(vector<hive::Table>* kudu_tables) {
Status HmsCatalog::AlterTable(const string& id,
const string& name,
const string& new_name,
- const Schema& schema) {
+ const Schema& schema,
+ const bool& check_id) {
Slice hms_database;
Slice hms_table;
RETURN_NOT_OK(ParseHiveTableIdentifier(name, &hms_database, &hms_table));
@@ -266,7 +267,7 @@ Status HmsCatalog::AlterTable(const string& id,
// Check that the HMS entry belongs to the table being altered.
if (table.parameters[HmsClient::kStorageHandlerKey] != HmsClient::kKuduStorageHandler ||
- table.parameters[HmsClient::kKuduTableIdKey] != id) {
+ (check_id && table.parameters[HmsClient::kKuduTableIdKey] != id)) {
// The original table isn't a Kudu table, or isn't the same Kudu table.
return Status::NotFound("the HMS entry for the table being "
"altered belongs to another table");
@@ -276,7 +277,7 @@ Status HmsCatalog::AlterTable(const string& id,
RETURN_NOT_OK(PopulateTable(id, new_name, table.owner, schema, master_addresses_,
table.tableType, &table));
return client->AlterTable(hms_database.ToString(), hms_table.ToString(),
- table, EnvironmentContext());
+ table, EnvironmentContext(check_id));
});
}
@@ -430,9 +431,12 @@ bool HmsCatalog::IsEnabled() {
return !FLAGS_hive_metastore_uris.empty();
}
-hive::EnvironmentContext HmsCatalog::EnvironmentContext() {
+hive::EnvironmentContext HmsCatalog::EnvironmentContext(const bool& check_id) {
hive::EnvironmentContext env_ctx;
- env_ctx.__set_properties({ std::make_pair(hms::HmsClient::kKuduMasterEventKey, "true") });
+ env_ctx.__set_properties({
+ std::make_pair(hms::HmsClient::kKuduMasterEventKey, "true"),
+ std::make_pair(hms::HmsClient::kKuduCheckIdKey, check_id ? "true" : "false"),
+ });
return env_ctx;
}
diff --git a/src/kudu/hms/hms_catalog.h b/src/kudu/hms/hms_catalog.h
index 22b2058..f4e92fd 100644
--- a/src/kudu/hms/hms_catalog.h
+++ b/src/kudu/hms/hms_catalog.h
@@ -88,12 +88,13 @@ class HmsCatalog {
// Alters a table entry in the HMS.
//
// This method will fail if the HMS is unreachable, if the table doesn't exist
- // in the HMS, or if the table entry in the HMS doesn't match the specified
- // Kudu table ID.
+ // in the HMS. It will also fail if check_id is true and the table entry in
+ // the HMS doesn't match the specified Kudu table ID.
Status AlterTable(const std::string& id,
const std::string& name,
const std::string& new_name,
- const Schema& schema) WARN_UNUSED_RESULT;
+ const Schema& schema,
+ const bool& check_id = true) WARN_UNUSED_RESULT;
// Upgrades a legacy Impala table entry in the HMS.
//
@@ -176,7 +177,7 @@ class HmsCatalog {
static Status ParseUris(const std::string& metastore_uris, std::vector<HostPort>* hostports);
// Returns a base environment context for use with calls to the HMS.
- static hive::EnvironmentContext EnvironmentContext();
+ static hive::EnvironmentContext EnvironmentContext(const bool& check_id = true);
// Kudu master addresses.
const std::string master_addresses_;
diff --git a/src/kudu/hms/hms_client.cc b/src/kudu/hms/hms_client.cc
index 679fd6a..61d87db 100644
--- a/src/kudu/hms/hms_client.cc
+++ b/src/kudu/hms/hms_client.cc
@@ -99,6 +99,7 @@ const char* const HmsClient::kKuduTableIdKey = "kudu.table_id";
const char* const HmsClient::kKuduTableNameKey = "kudu.table_name";
const char* const HmsClient::kKuduMasterAddrsKey = "kudu.master_addresses";
const char* const HmsClient::kKuduMasterEventKey = "kudu.master_event";
+const char* const HmsClient::kKuduCheckIdKey = "kudu.check_id";
const char* const HmsClient::kKuduStorageHandler = "org.apache.kudu.hive.KuduStorageHandler";
const char* const HmsClient::kTransactionalEventListeners =
diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h
index c21d612..67dcbed 100644
--- a/src/kudu/hms/hms_client.h
+++ b/src/kudu/hms/hms_client.h
@@ -63,6 +63,7 @@ class HmsClient {
static const char* const kKuduTableNameKey;
static const char* const kKuduMasterAddrsKey;
static const char* const kKuduMasterEventKey;
+ static const char* const kKuduCheckIdKey;
static const char* const kStorageHandlerKey;
static const char* const kKuduStorageHandler;
static const char* const kHiveFilterFieldParams;
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 727046a..471ef56 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -3467,6 +3467,14 @@ TEST_P(ToolTestKerberosParameterized, TestCheckAndAutomaticFixHmsMetadata) {
inconsistent_master_addrs->id(), inconsistent_master_addrs->name(), kUsername,
KuduSchema::ToSchema(inconsistent_master_addrs->schema())));
+ // Test case: bad table id.
+ shared_ptr<KuduTable> bad_id;
+ ASSERT_OK(CreateKuduTable(kudu_client, "default.bad_id"));
+ ASSERT_OK(kudu_client->OpenTable("default.bad_id", &bad_id));
+ ASSERT_OK(hms_catalog.CreateTable(
+ "not_a_table_id", "default.bad_id", kUsername,
+ KuduSchema::ToSchema(bad_id->schema())));
+
// Test cases: orphan tables in the HMS.
ASSERT_OK(hms_catalog.CreateTable(
"orphan-hms-table-id", "default.orphan_hms_table", kUsername,
@@ -3525,6 +3533,7 @@ TEST_P(ToolTestKerberosParameterized, TestCheckAndAutomaticFixHmsMetadata) {
"default.inconsistent_schema",
"default.inconsistent_name",
"default.inconsistent_master_addrs",
+ "default.bad_id",
"default.orphan_hms_table",
"default.orphan_hms_table_legacy_managed",
"default.kudu_orphan",
@@ -3610,6 +3619,7 @@ TEST_P(ToolTestKerberosParameterized, TestCheckAndAutomaticFixHmsMetadata) {
"default.inconsistent_schema",
"default.inconsistent_name",
"default.inconsistent_master_addrs",
+ "default.bad_id",
});
NO_FATALS(check());
@@ -3621,6 +3631,7 @@ TEST_P(ToolTestKerberosParameterized, TestCheckAndAutomaticFixHmsMetadata) {
"inconsistent_schema",
"inconsistent_name_hms",
"inconsistent_master_addrs",
+ "bad_id",
"kudu_orphan",
"legacy_managed",
"legacy_external",
@@ -3636,6 +3647,7 @@ TEST_P(ToolTestKerberosParameterized, TestCheckAndAutomaticFixHmsMetadata) {
kudu_client->ListTables(&kudu_tables);
std::sort(kudu_tables.begin(), kudu_tables.end());
ASSERT_EQ(vector<string>({
+ "default.bad_id",
"default.control",
"default.inconsistent_master_addrs",
"default.inconsistent_name_hms",
diff --git a/src/kudu/tools/tool_action_hms.cc b/src/kudu/tools/tool_action_hms.cc
index dfd2f69..1da8e74 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -394,14 +394,18 @@ Status AnalyzeCatalogs(const string& master_addrs,
// by table name.
shared_ptr<KuduTable>* kudu_table;
const string& storage_handler = hms_table.parameters[HmsClient::kStorageHandlerKey];
+ const string& hms_table_id = hms_table.parameters[HmsClient::kKuduTableIdKey];
+ const string& hms_table_name = hms_table.parameters[HmsClient::kKuduTableNameKey];
if (storage_handler == HmsClient::kKuduStorageHandler &&
hms_table.tableType == HmsClient::kManagedTable) {
- // TODO(ghenke): Also lookup by name to support tables created when HMS sync is off.
- // TODO(ghenke): Also lookup by name to support repairing id's out of sync.
- const string& hms_table_id = hms_table.parameters[HmsClient::kKuduTableIdKey];
kudu_table = FindOrNull(kudu_tables_by_id, hms_table_id);
+ // If there is no kudu table that matches the id, or the id doesn't exist,
+ // lookup the table by name. This handles manged tables created when HMS
+ // sync was off or tables with IDs out of sync.
+ if (!kudu_table) {
+ kudu_table = FindOrNull(kudu_tables_by_name, hms_table_name);
+ }
} else {
- const string& hms_table_name = hms_table.parameters[HmsClient::kKuduTableNameKey];
kudu_table = FindOrNull(kudu_tables_by_name, hms_table_name);
}
@@ -752,7 +756,9 @@ Status FixHmsMetadata(const RunnerContext& context) {
} else {
auto schema = KuduSchema::ToSchema(kudu_table.schema());
RETURN_NOT_OK_PREPEND(
- hms_catalog->AlterTable(kudu_table.id(), hms_table_name, hms_table_name, schema),
+ // Disable table ID checking to support fixing tables with unsynchronized IDs.
+ hms_catalog->AlterTable(kudu_table.id(), hms_table_name, hms_table_name, schema,
+ /* check_id */ false),
Substitute("failed to refresh HMS table metadata for Kudu table $0",
TableIdent(kudu_table)));
}