You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2019/04/21 20:58:41 UTC
[kudu] 01/02: Bump Sentry version
This is an automated email from the ASF dual-hosted git repository.
adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 345ce27da0d7d16ec37a56502d0939aef87f671f
Author: Hao Hao <ha...@cloudera.com>
AuthorDate: Mon Apr 15 22:43:32 2019 -0700
Bump Sentry version
The motivation is to get access to SENTRY-2471, which includes a fix for
syncing Sentry privilege changes upon table renaming even for tables
without HDFS path information. SENTRY-2471 has not been released yet,
so the new Sentry version is based on the latest commit of the master
branch.
Note that after bumping the Sentry version, the hms tests start to fail
with:
java.lang.UnsatisfiedLinkError: org.apache.hadoop.io.nativeio.NativeIO$POSIX.stat
(Ljava/lang/String;)Lorg/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat;
at org.apache.hadoop.io.nativeio.NativeIO$POSIX.stat(Native Method)
at org.apache.hadoop.io.nativeio.NativeIO$POSIX.getStat(NativeIO.java:451)
This is due to hadoop 3.1.1 (SENTRY-2329) from Sentry package is loaded
into the HMS classpath. Since we have no reasons to use hadoop jars in
Sentry package, we can safely remove them.
Change-Id: I13f05345d2e6dcdadd74503d0d4524947b137d43
Reviewed-on: http://gerrit.cloudera.org:8080/13059
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Hao Hao <ha...@cloudera.com>
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
src/kudu/integration-tests/master_sentry-itest.cc | 11 ++---------
src/kudu/sentry/mini_sentry.cc | 11 +++++++++++
src/kudu/sentry/sentry_common_service.thrift | 2 +-
src/kudu/sentry/sentry_policy_service.thrift | 18 +++++++++++++++---
thirdparty/build-thirdparty.sh | 4 ++++
thirdparty/vars.sh | 9 ++++++---
6 files changed, 39 insertions(+), 16 deletions(-)
diff --git a/src/kudu/integration-tests/master_sentry-itest.cc b/src/kudu/integration-tests/master_sentry-itest.cc
index b0f711d..0ada81f 100644
--- a/src/kudu/integration-tests/master_sentry-itest.cc
+++ b/src/kudu/integration-tests/master_sentry-itest.cc
@@ -447,15 +447,13 @@ TEST_F(MasterSentryTest, TestTableOwnership) {
NO_FATALS(CheckTableDoesNotExist(kDatabaseName, "new_table"));
}
-// TODO(hao): enable the following tests after SENTRY-2471 is fixed.
-TEST_F(MasterSentryTest, DISABLED_TestRenameTablePrivilegeTransfer) {
+// Checks Sentry privileges are synchronized upon table rename in the HMS.
+TEST_F(MasterSentryTest, TestRenameTablePrivilegeTransfer) {
ASSERT_OK(GrantRenameTablePrivilege(kDatabaseName, kTableName));
ASSERT_OK(RenameTable(Substitute("$0.$1", kDatabaseName, kTableName),
Substitute("$0.$1", kDatabaseName, "b")));
NO_FATALS(CheckTable(kDatabaseName, "b", make_optional<const string &>(kAdminUser)));
-
- // Checks table rename in the HMS is synchronized with the Sentry privileges.
unique_ptr<KuduTableAlterer> table_alterer;
table_alterer.reset(client_->NewTableAlterer(Substitute("$0.$1", kDatabaseName, "b"))
->DropColumn("int16_val"));
@@ -470,11 +468,6 @@ TEST_F(MasterSentryTest, DISABLED_TestRenameTablePrivilegeTransfer) {
ASSERT_OK(table_alterer->Alter());
});
NO_FATALS(CheckTable(kDatabaseName, "b", make_optional<const string&>(kAdminUser)));
-
- table_alterer.reset(client_->NewTableAlterer(Substitute("$0.$1", kDatabaseName, "b"))
- ->RenameTo(Substitute("$0.$1", kDatabaseName, "c")));
- ASSERT_OK(table_alterer->Alter());
- NO_FATALS(CheckTable(kDatabaseName, "c", make_optional<const string&>(kAdminUser)));
}
class TestAuthzTable :
diff --git a/src/kudu/sentry/mini_sentry.cc b/src/kudu/sentry/mini_sentry.cc
index 2477b8a..caecfa9 100644
--- a/src/kudu/sentry/mini_sentry.cc
+++ b/src/kudu/sentry/mini_sentry.cc
@@ -215,6 +215,12 @@ Status MiniSentry::CreateSentryConfigs(const string& tmp_dir) const {
// The interval to run the "store-cleaner" Sentry's thread. Setting to a
// negative value means Sentry will not run the "store-cleaner" thread
// at all and that allows for faster start-up times of the Sentry service.
+ //
+ // - hive.sentry.server
+ // Server namespace the HMS instance belongs to for defining server-level
+ // privileges in Sentry. Sentry uses it to synchronize privileges upon
+ // receipt of HMS events (such as table rename). Must match with Kudu
+ // master's flag 'server_name'.
static const string kFileTemplate = R"(
<configuration>
@@ -293,6 +299,11 @@ Status MiniSentry::CreateSentryConfigs(const string& tmp_dir) const {
<value>-1</value>
</property>
+ <property>
+ <name>hive.sentry.server</name>
+ <value>server1</value>
+ </property>
+
</configuration>
)";
diff --git a/src/kudu/sentry/sentry_common_service.thrift b/src/kudu/sentry/sentry_common_service.thrift
index b8a730e..a375348 100644
--- a/src/kudu/sentry/sentry_common_service.thrift
+++ b/src/kudu/sentry/sentry_common_service.thrift
@@ -19,7 +19,7 @@
*/
# DO NOT MODIFY! Copied from
-# https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_common_service.thrift
+# https://raw.githubusercontent.com/apache/sentry/505b42e81a9d85c4ebe8db3f48ad7a6e824a5db5/sentry-service/sentry-service-api/src/main/resources/sentry_common_service.thrift
#
# With edits:
# - Change cpp namespace to 'sentry' to match the Kudu codebase style.
diff --git a/src/kudu/sentry/sentry_policy_service.thrift b/src/kudu/sentry/sentry_policy_service.thrift
index dc2c13b..0d8c981 100644
--- a/src/kudu/sentry/sentry_policy_service.thrift
+++ b/src/kudu/sentry/sentry_policy_service.thrift
@@ -19,7 +19,7 @@
*/
# DO NOT MODIFY! Copied from
-# https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_policy_service.thrift
+# https://raw.githubusercontent.com/apache/sentry/505b42e81a9d85c4ebe8db3f48ad7a6e824a5db5/sentry-service/sentry-service-api/src/main/resources/sentry_policy_service.thrift
#
# With edits:
# - Change cpp namespace to 'sentry' to match the Kudu codebase style.
@@ -70,6 +70,15 @@ struct TSentryGroup {
1: required string groupName
}
+struct TIsSentryAdminRequest {
+1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
+2: required string userName,
+}
+struct TIsSentryAdminResponse {
+1: required sentry_common_service.TSentryResponseStatus status,
+2: required bool isAdmin,
+}
+
# CREATE ROLE r1
struct TCreateSentryRoleRequest {
1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
@@ -303,7 +312,7 @@ struct TSentryMappingData {
struct TSentryExportMappingDataRequest {
1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V1,
2: required string requestorUserName, # user on whose behalf the request is issued
-3: optional string objectPath # for specific auth object
+3: optional set<TSentryAuthorizable> authorizables # for which permission information needs to be exported.
}
struct TSentryExportMappingDataResponse {
@@ -314,7 +323,7 @@ struct TSentryExportMappingDataResponse {
struct TSentryImportMappingDataRequest {
1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V1,
2: required string requestorUserName, # user on whose behalf the request is issued
-3: required bool overwriteRole = false, # if overwrite the exist role with the imported privileges, default is false
+3: required bool overwriteRole = false, # if overwrite the exist role with the imported privileges, default is false
4: required TSentryMappingData mappingData
}
@@ -400,6 +409,9 @@ struct TSentryPrivilegesResponse {
service SentryPolicyService
{
+ # Check if the given user is in the Sentry admin group.
+ TIsSentryAdminResponse is_sentry_admin(1:TIsSentryAdminRequest request)
+
TCreateSentryRoleResponse create_sentry_role(1:TCreateSentryRoleRequest request)
TDropSentryRoleResponse drop_sentry_role(1:TDropSentryRoleRequest request)
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index c13c95c..0fe31ca 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -259,6 +259,10 @@ fi
if [ -n "$F_COMMON" -o -n "$F_SENTRY" ]; then
mkdir -p $PREFIX/opt
+ # Remove any hadoop jars included in the Sentry package to avoid unexpected
+ # runtime behavior, due to different versions of hadoop jars are loaded
+ # (one from Kudu's third-party dependency, the other from the Sentry package).
+ rm -rf $SENTRY_SOURCE/lib/hadoop-[a-z-]*.jar
ln -nsf $SENTRY_SOURCE $PREFIX/opt/sentry
fi
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 1c142a7..ba0b700 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -226,8 +226,11 @@ HADOOP_VERSION=2.8.5
HADOOP_NAME=hadoop-$HADOOP_VERSION
HADOOP_SOURCE=$TP_SOURCE_DIR/$HADOOP_NAME
-# TODO(dan): bump to a release version once SENTRY-2371 is published. The SHA
-# below is the current head of the master branch.
-SENTRY_VERSION=2c9a927a9e87cba0e4c0f34fc0b55887c6636927
+# TODO(dan): bump to a release version once SENTRY-2371, SENTRY-2440 and SENTRY-2471
+# are published. The SHA below is the current head of the master branch.
+# Note: Sentry releases source code only. To build the binary tarball, use `dist`
+# maven profile. For example, `mvn clean install -Pdist`. After a successful build,
+# the tarball will be available under sentry-dist/target.
+SENTRY_VERSION=505b42e81a9d85c4ebe8db3f48ad7a6e824a5db5
SENTRY_NAME=apache-sentry-$SENTRY_VERSION-bin
SENTRY_SOURCE=$TP_SOURCE_DIR/$SENTRY_NAME