You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/06/15 21:40:04 UTC

[1/2] kudu git commit: KUDU-2191: Add a method to decide the scope of AlterTable

Repository: kudu
Updated Branches:
  refs/heads/master 5f1ca32f3 -> f2089195d


KUDU-2191: Add a method to decide the scope of AlterTable

This commit adds a private method in KuduTableAlterer to indicate
whether the table alteration should be applied to external catalogs,
such as the Hive Metastore, which the Kudu master may has been
configured to integrate with.

This method will be useful to the HMS integration tools, such as the
metadata upgrade tool which may need to alter tables in Kudu only.

Change-Id: I703f12b95bb7ca2d65455f0f0602520332b3c678
Reviewed-on: http://gerrit.cloudera.org:8080/10581
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao <ha...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e3474f4c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e3474f4c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e3474f4c

Branch: refs/heads/master
Commit: e3474f4ce7b1ebc88d12c8ffe1837df8e8211603
Parents: 5f1ca32
Author: hahao <ha...@cloudera.com>
Authored: Fri Jun 1 15:54:28 2018 -0700
Committer: Hao Hao <ha...@cloudera.com>
Committed: Fri Jun 15 18:18:40 2018 +0000

----------------------------------------------------------------------
 src/kudu/client/client.cc                      | 10 ++++++++--
 src/kudu/client/client.h                       |  8 ++++++++
 src/kudu/client/table_alterer-internal.cc      |  1 +
 src/kudu/client/table_alterer-internal.h       |  5 ++++-
 src/kudu/integration-tests/master_hms-itest.cc | 10 ++++++++++
 src/kudu/master/catalog_manager.cc             | 11 ++++++-----
 src/kudu/master/master.proto                   |  4 ++++
 7 files changed, 41 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/client.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc
index 87851e0..912e71a 100644
--- a/src/kudu/client/client.cc
+++ b/src/kudu/client/client.cc
@@ -396,8 +396,8 @@ Status KuduClient::DeleteTable(const string& table_name) {
   return data_->DeleteTable(this, table_name, deadline);
 }
 
-KuduTableAlterer* KuduClient::NewTableAlterer(const string& name) {
-  return new KuduTableAlterer(this, name);
+KuduTableAlterer* KuduClient::NewTableAlterer(const string& table_name) {
+  return new KuduTableAlterer(this, table_name);
 }
 
 Status KuduClient::IsAlterTableInProgress(const string& table_name,
@@ -1140,6 +1140,12 @@ KuduTableAlterer* KuduTableAlterer::wait(bool wait) {
   return this;
 }
 
+KuduTableAlterer* KuduTableAlterer::alter_external_catalogs(
+    bool alter_external_catalogs) {
+  data_->alter_external_catalogs_ = alter_external_catalogs;
+  return this;
+}
+
 Status KuduTableAlterer::Alter() {
   AlterTableRequestPB req;
   AlterTableResponsePB resp;

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/client.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index 852c7be..3404b9a 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -50,6 +50,7 @@ namespace kudu {
 
 class ClientStressTest_TestUniqueClientIds_Test;
 class KuduPartialRow;
+class MasterHmsTest_TestAlterTable_Test;
 class MonoDelta;
 class PartitionSchema;
 class SecurityUnknownTskTest;
@@ -1182,9 +1183,16 @@ class KUDU_EXPORT KuduTableAlterer {
   class KUDU_NO_EXPORT Data;
   friend class KuduClient;
 
+  FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable);
+
   KuduTableAlterer(KuduClient* client,
                    const std::string& name);
 
+  // Whether to apply the alteration to external catalogs, such as the Hive
+  // Metastore, which the Kudu master has been configured to integrate with.
+  // This method returns a raw pointer to this alterer object.
+  KuduTableAlterer* alter_external_catalogs(bool alter_external_catalogs);
+
   // Owned.
   Data* data_;
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/table_alterer-internal.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/table_alterer-internal.cc b/src/kudu/client/table_alterer-internal.cc
index 2baebdf..12adcaf 100644
--- a/src/kudu/client/table_alterer-internal.cc
+++ b/src/kudu/client/table_alterer-internal.cc
@@ -61,6 +61,7 @@ Status KuduTableAlterer::Data::ToRequest(AlterTableRequestPB* req) {
   }
 
   req->Clear();
+  req->set_alter_external_catalogs(alter_external_catalogs_);
   req->mutable_table()->set_table_name(table_name_);
   if (rename_to_.is_initialized()) {
     req->set_new_table_name(rename_to_.get());

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/table_alterer-internal.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/table_alterer-internal.h b/src/kudu/client/table_alterer-internal.h
index f847ace..6f2e4c3 100644
--- a/src/kudu/client/table_alterer-internal.h
+++ b/src/kudu/client/table_alterer-internal.h
@@ -44,7 +44,6 @@ class KuduTableAlterer::Data {
   ~Data();
   Status ToRequest(master::AlterTableRequestPB* req);
 
-
   KuduClient* const client_;
   const std::string table_name_;
 
@@ -78,6 +77,10 @@ class KuduTableAlterer::Data {
   // Schema of add/drop range partition bound rows.
   const Schema* schema_;
 
+  // Whether to apply the alteration to external catalogs, such as the Hive
+  // Metastore. The default value is true.
+  bool alter_external_catalogs_ = true;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(Data);
 };

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/integration-tests/master_hms-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/master_hms-itest.cc b/src/kudu/integration-tests/master_hms-itest.cc
index 68c6728..160121d 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -347,6 +347,16 @@ TEST_F(MasterHmsTest, TestAlterTable) {
     ASSERT_OK(table_alterer->Alter());
   });
   NO_FATALS(CheckTable("default", "a"));
+
+  // Only alter the table in Kudu, the corresponding table in the HMS will not be altered.
+  table_alterer.reset(client_->NewTableAlterer("default.a")->RenameTo("default.b")
+                             ->alter_external_catalogs(false));
+  ASSERT_OK(table_alterer->Alter());
+  bool exists;
+  ASSERT_OK(client_->TableExists("default.b", &exists));
+  ASSERT_TRUE(exists);
+  hive::Table hms_table;
+  ASSERT_OK(hms_client_->GetTable("default", "a", &hms_table));
 }
 
 TEST_F(MasterHmsTest, TestDeleteTable) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/master/catalog_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 9e8d692..49e368b 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -2136,12 +2136,13 @@ Status CatalogManager::AlterTableRpc(const AlterTableRequestPB& req,
   LOG(INFO) << Substitute("Servicing AlterTable request from $0:\n$1",
                           RequestorString(rpc), SecureShortDebugString(req));
 
-  // If the HMS integration is enabled and the alteration includes a table
-  // rename, then don't directly rename the table in the Kudu catalog. Instead,
-  // rename the table in the HMS and wait for the notification log listener to
-  // apply that event to the catalog. By 'serializing' the rename through the
+  // If the HMS integration is enabled, the alteration includes a table
+  // rename and the table should be altered in the HMS, then don't directly
+  // rename the table in the Kudu catalog. Instead, rename the table
+  // in the HMS and wait for the notification log listener to apply
+  // that event to the catalog. By 'serializing' the rename through the
   // HMS, race conditions are avoided.
-  if (hms_catalog_ && req.has_new_table_name()) {
+  if (hms_catalog_ && req.has_new_table_name() && req.alter_external_catalogs()) {
     // Look up the table, lock it, and mark it as removed.
     scoped_refptr<TableInfo> table;
     TableMetadataLock l;

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/master/master.proto
----------------------------------------------------------------------
diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto
index 1b59ef0..6fba23b 100644
--- a/src/kudu/master/master.proto
+++ b/src/kudu/master/master.proto
@@ -564,6 +564,10 @@ message AlterTableRequestPB {
   // The table schema to use when decoding the range bound row operations. Only
   // necessary when partitions are being added or dropped.
   optional SchemaPB schema = 4;
+
+  // Whether to apply the alteration to external catalogs, such as the Hive Metastore,
+  // which the Kudu master has been configured to integrate with.
+  optional bool alter_external_catalogs = 5 [default = true];
 }
 
 message AlterTableResponsePB {


[2/2] kudu git commit: KUDU-2191: allow metadata upgrade tool to run with HMS integration

Posted by gr...@apache.org.
KUDU-2191: allow metadata upgrade tool to run with HMS integration

Previously the metadata upgrade tool requires the HMS integration
feature being disabled. This commit updates the tool to only alter
tables in Kudu but not in the HMS during upgrade, so that the
restriction no longer applies.

Change-Id: I51d4cabeb1a9defc51f4e307a116419da5588f2d
Reviewed-on: http://gerrit.cloudera.org:8080/10582
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f2089195
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f2089195
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f2089195

Branch: refs/heads/master
Commit: f2089195d635ea73e7268a9b46db2feac2eeed2d
Parents: e3474f4
Author: hahao <ha...@cloudera.com>
Authored: Wed Jun 6 23:58:49 2018 -0700
Committer: Hao Hao <ha...@cloudera.com>
Committed: Fri Jun 15 20:07:45 2018 +0000

----------------------------------------------------------------------
 src/kudu/client/client.h          | 16 +++++++++++++++-
 src/kudu/tools/kudu-tool-test.cc  |  5 +++++
 src/kudu/tools/tool_action_hms.cc | 21 ++++++++++++---------
 3 files changed, 32 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/client/client.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index 3404b9a..f03c0e3 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -37,6 +37,7 @@
 #include "kudu/client/shared_ptr.h" // IWYU pragma: keep
 #ifdef KUDU_HEADERS_NO_STUBS
 #include <gtest/gtest_prod.h>
+
 #include "kudu/gutil/macros.h"
 #include "kudu/gutil/port.h"
 #else
@@ -55,13 +56,19 @@ class MonoDelta;
 class PartitionSchema;
 class SecurityUnknownTskTest;
 
+namespace client {
+class KuduClient;
+}
+
 namespace tools {
 class LeaderMasterProxy;
+
+Status AlterKuduTable(const client::sp::shared_ptr<client::KuduClient>& kudu_client,
+                      const std::string& name, const std::string& new_name);
 } // namespace tools
 
 namespace client {
 
-class KuduClient;
 class KuduDelete;
 class KuduInsert;
 class KuduLoggingCallback;
@@ -1181,8 +1188,14 @@ class KUDU_EXPORT KuduTableAlterer {
 
  private:
   class KUDU_NO_EXPORT Data;
+
   friend class KuduClient;
 
+  friend Status tools::AlterKuduTable(
+      const client::sp::shared_ptr<client::KuduClient>& kudu_client,
+      const std::string& name,
+      const std::string& new_name);
+
   FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable);
 
   KuduTableAlterer(KuduClient* client,
@@ -2391,6 +2404,7 @@ class KUDU_EXPORT KuduPartitioner {
   Status PartitionRow(const KuduPartialRow& row, int* partition);
  private:
   class KUDU_NO_EXPORT Data;
+
   friend class KuduPartitionerBuilder;
 
   explicit KuduPartitioner(Data* data);

http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/tools/kudu-tool-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 12bd979..57bc295 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -2075,6 +2075,11 @@ TEST_F(ToolTest, TestHmsUpgrade) {
     ASSERT_EQ(2, table_names.size());
   }
 
+  // Restart external mini cluster to enable Hive Metastore integration.
+  cluster_->EnableMetastoreIntegration();
+  cluster_->ShutdownNodes(cluster::ClusterNodes::ALL);
+  ASSERT_OK(cluster_->Restart());
+
   // Upgrade the historical metadata in both Hive Metastore and Kudu.
   string out;
   NO_FATALS(RunActionStdinStdoutString(

http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/tools/tool_action_hms.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_hms.cc b/src/kudu/tools/tool_action_hms.cc
index 1f141c0..5c90c44 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -102,6 +102,16 @@ string RenameHiveIncompatibleTable(const string& name) {
   return table_name;
 }
 
+// Only alter the table in Kudu but not in the Hive Metastore.
+Status AlterKuduTable(const client::sp::shared_ptr<KuduClient>& kudu_client,
+                      const string& name,
+                      const string& new_name) {
+  unique_ptr<KuduTableAlterer> alterer(kudu_client->NewTableAlterer(name));
+  return alterer->RenameTo(new_name)
+                ->alter_external_catalogs(false)
+                ->Alter();
+}
+
 // Alter legacy tables (which includes non-Impala tables, Impala managed/external
 // tables) to follow the format 'database_name.table_name' in table naming in Kudu.
 // Also, create HMS entries for non-Impala tables.
@@ -126,13 +136,6 @@ Status AlterLegacyKuduTables(const client::sp::shared_ptr<KuduClient>& kudu_clie
     }
   }
 
-  auto alter_kudu_table = [&](const string& name,
-                              const string& new_name) -> Status {
-    unique_ptr<KuduTableAlterer> alterer(kudu_client->NewTableAlterer(name));
-    return alterer->RenameTo(new_name)
-                  ->Alter();
-  };
-
   unordered_map<string, Status> failures;
   for (const auto& table_name : table_names) {
     hive::Table* hms_table = FindOrNull(hms_tables_map, table_name);
@@ -150,7 +153,7 @@ Status AlterLegacyKuduTables(const client::sp::shared_ptr<KuduClient>& kudu_clie
         RETURN_NOT_OK(kudu_client->TableExists(new_table_name, &exist));
         if (!exist) {
           // TODO(Hao): Use notification listener to avoid race conditions.
-          s = alter_kudu_table(table_name, new_table_name).AndThen([&] {
+          s = AlterKuduTable(kudu_client, table_name, new_table_name).AndThen([&] {
             return hms_catalog->UpgradeLegacyImpalaTable(kudu_table->id(),
                 hms_table->dbName, hms_table->tableName,
                 client::SchemaFromKuduSchema(kudu_table->schema()));
@@ -175,7 +178,7 @@ Status AlterLegacyKuduTables(const client::sp::shared_ptr<KuduClient>& kudu_clie
         s = hms_catalog->CreateTable(kudu_table->id(), new_table_name, schema);
       }
       s = s.AndThen([&] {
-        return alter_kudu_table(table_name, new_table_name);
+        return AlterKuduTable(kudu_client, table_name, new_table_name);
       });
     }