You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/10/03 21:40:55 UTC
[2/3] kudu git commit: [hms] enable
'notifications.add.thrift.objects' in mini HMS
[hms] enable 'notifications.add.thrift.objects' in mini HMS
This commits explictly enables 'notifications.add.thrift.objects' to
add the entire thrift Table/Partition objects to Hive notification log.
So that Kudu can get all the required fields to properly parse the
notification messages, when running with Hive distributions that
disable it by default.
Change-Id: I9f28f52bdddd59c5b0dc7120d62482de8becb1ee
Reviewed-on: http://gerrit.cloudera.org:8080/11566
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ce2b5277
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ce2b5277
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ce2b5277
Branch: refs/heads/master
Commit: ce2b52770be4a81800fb5eb155b952d5211e2017
Parents: 63e4a0a
Author: Hao Hao <ha...@cloudera.com>
Authored: Mon Oct 1 13:05:27 2018 -0700
Committer: Hao Hao <ha...@cloudera.com>
Committed: Wed Oct 3 18:24:32 2018 +0000
----------------------------------------------------------------------
src/kudu/hms/hms_client.cc | 22 ++++++++++++++++++++--
src/kudu/hms/hms_client.h | 1 +
src/kudu/hms/mini_hms.cc | 9 +++++++++
3 files changed, 30 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/ce2b5277/src/kudu/hms/hms_client.cc
----------------------------------------------------------------------
diff --git a/src/kudu/hms/hms_client.cc b/src/kudu/hms/hms_client.cc
index ef37616..9d51956 100644
--- a/src/kudu/hms/hms_client.cc
+++ b/src/kudu/hms/hms_client.cc
@@ -112,6 +112,8 @@ const char* const HmsClient::kStorageHandlerKey = "storage_handler";
const char* const HmsClient::kKuduMetastorePlugin =
"org.apache.kudu.hive.metastore.KuduMetastorePlugin";
const char* const HmsClient::kHiveFilterFieldParams = "hive_filter_field_params__";
+const char* const HmsClient::kNotificationAddThriftObjects =
+ "hive.metastore.notifications.add.thrift.objects";
const char* const HmsClient::kManagedTable = "MANAGED_TABLE";
const char* const HmsClient::kExternalTable = "EXTERNAL_TABLE";
@@ -174,10 +176,26 @@ Status HmsClient::Start() {
if (boost::iequals(disallow_incompatible_column_type_changes, "true")) {
return Status::IllegalState(Substitute(
- "Hive Metastore configuration is invalid: $0 must be set to false",
- kDisallowIncompatibleColTypeChanges));
+ "Hive Metastore configuration is invalid: $0 must be set to false",
+ kDisallowIncompatibleColTypeChanges));
}
+ // Check that the HMS is configured to add the entire thrift Table/Partition
+ // objects to the HMS notifications, which is required to properly parse the
+ // HMS notification log. This is specific to the HMS version shipped in
+ // Cloudera's CDH.
+ string thrift_objects_config;
+ HMS_RET_NOT_OK(client_.get_config_value(thrift_objects_config,
+ kNotificationAddThriftObjects,
+ "true"),
+ Substitute("failed to get Hive Metastore $0 configuration",
+ kNotificationAddThriftObjects));
+ if (boost::iequals(thrift_objects_config, "false")) {
+ return Status::IllegalState(Substitute(
+ "Hive Metastore configuration is invalid: $0 must be set to true",
+ kNotificationAddThriftObjects));
+ }
+
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/ce2b5277/src/kudu/hms/hms_client.h
----------------------------------------------------------------------
diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h
index c081366..23451f3 100644
--- a/src/kudu/hms/hms_client.h
+++ b/src/kudu/hms/hms_client.h
@@ -83,6 +83,7 @@ class HmsClient {
static const char* const kDisallowIncompatibleColTypeChanges;
static const char* const kDbNotificationListener;
static const char* const kKuduMetastorePlugin;
+ static const char* const kNotificationAddThriftObjects;
// See org.apache.hadoop.hive.metastore.TableType.
static const char* const kManagedTable;
http://git-wip-us.apache.org/repos/asf/kudu/blob/ce2b5277/src/kudu/hms/mini_hms.cc
----------------------------------------------------------------------
diff --git a/src/kudu/hms/mini_hms.cc b/src/kudu/hms/mini_hms.cc
index 253de09..079cdb1 100644
--- a/src/kudu/hms/mini_hms.cc
+++ b/src/kudu/hms/mini_hms.cc
@@ -207,6 +207,10 @@ Status MiniHms::CreateHiveSite() const {
// - hive.support.special.characters.tablename
// Configures the HMS to allow special characters such as '/' in table
// names.
+ //
+ // - hive.metastore.notifications.add.thrift.objects
+ // Configured the HMS to add the entire thrift Table/Partition
+ // objects to the HMS notifications.
static const string kFileTemplate = R"(
<configuration>
<property>
@@ -271,6 +275,11 @@ Status MiniHms::CreateHiveSite() const {
<name>hive.support.special.characters.tablename</name>
<value>true</value>
</property>
+
+ <property>
+ <name>hive.metastore.notifications.add.thrift.objects</name>
+ <value>true</value>
+ </property>
</configuration>
)";