You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/02/16 01:20:55 UTC
[impala] 05/11: IMPALA-5552: Add support for authorized proxy groups
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch 2.x
in repository https://gitbox.apache.org/repos/asf/impala.git
commit b8e995e042619c6062cfa970b21924508e16677d
Author: Fredy Wijaya <fw...@cloudera.com>
AuthorDate: Wed May 23 20:16:00 2018 -0700
IMPALA-5552: Add support for authorized proxy groups
The patch adds support for mapping of users to a list of proxy groups.
The following flags are added in impalad:
- authorized_proxy_group_config
- authorized_proxy_group_config_delimiter
Example:
--authorized_proxy_group_config=hue=group1,group2;user1=*
This feature is not supported on Shell-based Hadoop groups mapping
providers.
The authorized_proxy_user/group_config parser will now strip leading
and trailing whitespaces from the user/group names.
Testing:
- Added FE unit test to check for groups mapping provider
- Added BE unit test for the parsing logic
- Added a new test in test_authorization.py
- Ran all end-to-end test_authorization.py
Change-Id: I6953f89c293b06b72f523b11802232133d9d6cbb
Reviewed-on: http://gerrit.cloudera.org:8080/10510
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
---
be/src/service/CMakeLists.txt | 1 +
be/src/service/frontend.cc | 16 +++
be/src/service/frontend.h | 5 +
be/src/service/impala-server-test.cc | 74 +++++++++++++
be/src/service/impala-server.cc | 117 ++++++++++++++++-----
be/src/service/impala-server.h | 23 +++-
be/src/util/backend-gflag-util.cc | 3 +
common/thrift/BackendGflags.thrift | 2 +
common/thrift/Frontend.thrift | 10 ++
.../org/apache/impala/service/BackendConfig.java | 4 +
.../org/apache/impala/service/JniFrontend.java | 67 +++++++++++-
.../org/apache/impala/service/JniFrontendTest.java | 58 ++++++++++
tests/authorization/test_authorization.py | 40 ++++++-
13 files changed, 384 insertions(+), 36 deletions(-)
diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt
index c78116c..66cf55d 100644
--- a/be/src/service/CMakeLists.txt
+++ b/be/src/service/CMakeLists.txt
@@ -77,3 +77,4 @@ target_link_libraries(impalad
ADD_BE_TEST(session-expiry-test session-expiry-test.cc)
ADD_BE_TEST(hs2-util-test hs2-util-test.cc)
ADD_BE_TEST(query-options-test query-options-test.cc)
+ADD_BE_TEST(impala-server-test impala-server-test.cc)
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index ecfae0a..0108f63 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -57,6 +57,16 @@ DEFINE_string(authorized_proxy_user_config, "",
"users. For example: hue=user1,user2;admin=*");
DEFINE_string(authorized_proxy_user_config_delimiter, ",",
"Specifies the delimiter used in authorized_proxy_user_config. ");
+DEFINE_string(authorized_proxy_group_config, "",
+ "Specifies the set of authorized proxy groups (users who can delegate to other "
+ "users belonging to the specified groups during authorization) and whom they are "
+ "allowed to delegate. Input is a semicolon-separated list of key=value pairs of "
+ "authorized proxy users to the group(s) they can delegate to. These groups are "
+ "specified as a list of groups separated by a delimiter (which defaults to comma and "
+ "may be changed via --authorized_proxy_group_config_delimiter), or '*' to indicate "
+ "all users. For example: hue=group1,group2;admin=*");
+DEFINE_string(authorized_proxy_group_config_delimiter, ",",
+ "Specifies the delimiter used in authorized_proxy_group_config. ");
DEFINE_string(kudu_master_hosts, "", "Specifies the default Kudu master(s). The given "
"value should be a comma separated list of hostnames or IP addresses; ports are "
"optional.");
@@ -68,6 +78,7 @@ Frontend::Frontend() {
{"getExplainPlan", "([B)Ljava/lang/String;", &get_explain_plan_id_},
{"getHadoopConfig", "([B)[B", &get_hadoop_config_id_},
{"getAllHadoopConfigs", "()[B", &get_hadoop_configs_id_},
+ {"getHadoopGroups", "([B)[B", &get_hadoop_groups_id_},
{"checkConfiguration", "()Ljava/lang/String;", &check_config_id_},
{"updateCatalogCache", "([B)[B", &update_catalog_cache_id_},
{"updateMembership", "([B)V", &update_membership_id_},
@@ -242,6 +253,11 @@ Status Frontend::GetHadoopConfig(const TGetHadoopConfigRequest& request,
return JniUtil::CallJniMethod(fe_, get_hadoop_config_id_, request, response);
}
+Status Frontend::GetHadoopGroups(const TGetHadoopGroupsRequest& request,
+ TGetHadoopGroupsResponse* response) {
+ return JniUtil::CallJniMethod(fe_, get_hadoop_groups_id_, request, response);
+}
+
Status Frontend::LoadData(const TLoadDataReq& request, TLoadDataResp* response) {
return JniUtil::CallJniMethod(fe_, load_table_data_id_, request, response);
}
diff --git a/be/src/service/frontend.h b/be/src/service/frontend.h
index 9198b83..08123b3 100644
--- a/be/src/service/frontend.h
+++ b/be/src/service/frontend.h
@@ -153,6 +153,10 @@ class Frontend {
Status GetHadoopConfig(const TGetHadoopConfigRequest& request,
TGetHadoopConfigResponse* response);
+ /// Returns (in the output parameter) the list of groups for the given user.
+ Status GetHadoopGroups(const TGetHadoopGroupsRequest& request,
+ TGetHadoopGroupsResponse* response);
+
/// Loads a single file or set of files into a table or partition. Saves the RPC
/// response in the TLoadDataResp output parameter. Returns OK if the operation
/// completed successfully.
@@ -186,6 +190,7 @@ class Frontend {
jmethodID get_explain_plan_id_; // JniFrontend.getExplainPlan()
jmethodID get_hadoop_config_id_; // JniFrontend.getHadoopConfig(byte[])
jmethodID get_hadoop_configs_id_; // JniFrontend.getAllHadoopConfigs()
+ jmethodID get_hadoop_groups_id_; // JniFrontend.getHadoopGroups()
jmethodID check_config_id_; // JniFrontend.checkConfiguration()
jmethodID update_catalog_cache_id_; // JniFrontend.updateCatalogCache(byte[][])
jmethodID update_membership_id_; // JniFrontend.updateMembership()
diff --git a/be/src/service/impala-server-test.cc b/be/src/service/impala-server-test.cc
new file mode 100644
index 0000000..772e70f
--- /dev/null
+++ b/be/src/service/impala-server-test.cc
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gutil/strings/substitute.h>
+#include <sstream>
+#include <vector>
+
+#include "service/impala-server.h"
+#include "testutil/gtest-util.h"
+
+using namespace impala;
+using namespace std;
+using namespace strings;
+
+namespace impala {
+
+using AuthorizedProxyMap =
+ boost::unordered_map<std::string, boost::unordered_set<std::string>>;
+
+class ImpalaServerTest : public testing::Test {
+public:
+ static Status PopulateAuthorizedProxyConfig(
+ const string& authorized_proxy_config,
+ const string& authorized_proxy_config_delimiter,
+ AuthorizedProxyMap* authorized_proxy_map) {
+ return ImpalaServer::PopulateAuthorizedProxyConfig(authorized_proxy_config,
+ authorized_proxy_config_delimiter, authorized_proxy_map);
+ }
+};
+
+}
+
+TEST(ImpalaServerTest, PopulateAuthorizedProxyConfig) {
+ vector<string> delimiters{",", "@", " "};
+ for (auto& delimiter : delimiters) {
+ AuthorizedProxyMap proxy_map;
+ Status status = ImpalaServerTest::PopulateAuthorizedProxyConfig(
+ Substitute("hue=user1$0user2;impala = user3 ;hive=* ", delimiter), delimiter,
+ &proxy_map);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(3ul, proxy_map.size());
+
+ auto proxies = proxy_map["hue"];
+ EXPECT_EQ(2ul, proxies.size());
+ EXPECT_EQ("user1", *proxies.find("user1"));
+ EXPECT_EQ("user2", *proxies.find("user2"));
+
+ proxies = proxy_map["impala"];
+ EXPECT_EQ(1ul, proxies.size());
+ EXPECT_EQ("user3", *proxies.find("user3"));
+
+ proxies = proxy_map["hive"];
+ EXPECT_EQ(1ul, proxies.size());
+ EXPECT_EQ("*", *proxies.find("*"));
+
+ EXPECT_EQ(proxy_map.end(), proxy_map.find("doesnotexist"));
+ }
+}
+
+IMPALA_TEST_MAIN();
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index 2259791..771b0eb 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -26,6 +26,7 @@
#include <boost/unordered_set.hpp>
#include <boost/bind.hpp>
#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/trim.hpp>
#include <boost/lexical_cast.hpp>
#include <gperftools/malloc_extension.h>
#include <gutil/strings/substitute.h>
@@ -113,6 +114,8 @@ DECLARE_string(nn);
DECLARE_int32(nn_port);
DECLARE_string(authorized_proxy_user_config);
DECLARE_string(authorized_proxy_user_config_delimiter);
+DECLARE_string(authorized_proxy_group_config);
+DECLARE_string(authorized_proxy_group_config_delimiter);
DECLARE_bool(abort_on_config_error);
DECLARE_bool(disk_spill_encryption);
DECLARE_bool(use_krpc);
@@ -316,29 +319,22 @@ ImpalaServer::ImpalaServer(ExecEnv* exec_env)
}
if (!FLAGS_authorized_proxy_user_config.empty()) {
- // Parse the proxy user configuration using the format:
- // <proxy user>=<comma separated list of users they are allowed to delegate>
- // See FLAGS_authorized_proxy_user_config for more details.
- vector<string> proxy_user_config;
- split(proxy_user_config, FLAGS_authorized_proxy_user_config, is_any_of(";"),
- token_compress_on);
- if (proxy_user_config.size() > 0) {
- for (const string& config: proxy_user_config) {
- size_t pos = config.find("=");
- if (pos == string::npos) {
- CLEAN_EXIT_WITH_ERROR(Substitute("Invalid proxy user configuration. No "
- "mapping value specified for the proxy user. For more information review "
- "usage of the --authorized_proxy_user_config flag: $0", config));
- }
- string proxy_user = config.substr(0, pos);
- string config_str = config.substr(pos + 1);
- vector<string> parsed_allowed_users;
- split(parsed_allowed_users, config_str,
- is_any_of(FLAGS_authorized_proxy_user_config_delimiter), token_compress_on);
- unordered_set<string> allowed_users(parsed_allowed_users.begin(),
- parsed_allowed_users.end());
- authorized_proxy_user_config_.insert(make_pair(proxy_user, allowed_users));
- }
+ Status status = PopulateAuthorizedProxyConfig(FLAGS_authorized_proxy_user_config,
+ FLAGS_authorized_proxy_user_config_delimiter, &authorized_proxy_user_config_);
+ if (!status.ok()) {
+ CLEAN_EXIT_WITH_ERROR(Substitute("Invalid proxy user configuration."
+ "No mapping value specified for the proxy user. For more information review "
+ "usage of the --authorized_proxy_user_config flag: $0", status.GetDetail()));
+ }
+ }
+
+ if (!FLAGS_authorized_proxy_group_config.empty()) {
+ Status status = PopulateAuthorizedProxyConfig(FLAGS_authorized_proxy_group_config,
+ FLAGS_authorized_proxy_group_config_delimiter, &authorized_proxy_group_config_);
+ if (!status.ok()) {
+ CLEAN_EXIT_WITH_ERROR(Substitute("Invalid proxy group configuration. "
+ "No mapping value specified for the proxy group. For more information review "
+ "usage of the --authorized_proxy_group_config flag: $0", status.GetDetail()));
}
}
@@ -403,6 +399,38 @@ ImpalaServer::ImpalaServer(ExecEnv* exec_env)
exec_env_->SetImpalaServer(this);
}
+Status ImpalaServer::PopulateAuthorizedProxyConfig(
+ const string& authorized_proxy_config,
+ const string& authorized_proxy_config_delimiter,
+ AuthorizedProxyMap* authorized_proxy_config_map) {
+ // Parse the proxy user configuration using the format:
+ // <proxy user>=<comma separated list of users/groups they are allowed to delegate>
+ // See FLAGS_authorized_proxy_user_config or FLAGS_authorized_proxy_group_config
+ // for more details.
+ vector<string> proxy_config;
+ split(proxy_config, authorized_proxy_config, is_any_of(";"),
+ token_compress_on);
+ if (proxy_config.size() > 0) {
+ for (const string& config: proxy_config) {
+ size_t pos = config.find("=");
+ if (pos == string::npos) {
+ return Status(config);
+ }
+ string proxy_user = config.substr(0, pos);
+ boost::trim(proxy_user);
+ string config_str = config.substr(pos + 1);
+ boost::trim(config_str);
+ vector<string> parsed_allowed_users_or_groups;
+ split(parsed_allowed_users_or_groups, config_str,
+ is_any_of(authorized_proxy_config_delimiter), token_compress_on);
+ unordered_set<string> allowed_users_or_groups(
+ parsed_allowed_users_or_groups.begin(), parsed_allowed_users_or_groups.end());
+ authorized_proxy_config_map->insert({proxy_user, allowed_users_or_groups});
+ }
+ }
+ return Status::OK();
+}
+
Status ImpalaServer::LogLineageRecord(const ClientRequestState& client_request_state) {
const TExecRequest& request = client_request_state.exec_request();
if (!request.__isset.query_exec_request && !request.__isset.catalog_op_request) {
@@ -1341,8 +1369,9 @@ Status ImpalaServer::AuthorizeProxyUser(const string& user, const string& do_as_
stringstream error_msg;
error_msg << "User '" << user << "' is not authorized to delegate to '"
<< do_as_user << "'.";
- if (authorized_proxy_user_config_.size() == 0) {
- error_msg << " User delegation is disabled.";
+ if (authorized_proxy_user_config_.size() == 0 &&
+ authorized_proxy_group_config_.size() == 0) {
+ error_msg << " User/group delegation is disabled.";
VLOG(1) << error_msg;
return Status::Expected(error_msg.str());
}
@@ -1357,13 +1386,45 @@ Status ImpalaServer::AuthorizeProxyUser(const string& user, const string& do_as_
// Check if the proxy user exists. If he/she does, then check if they are allowed
// to delegate to the do_as_user.
- ProxyUserMap::const_iterator proxy_user =
+ AuthorizedProxyMap::const_iterator proxy_user =
authorized_proxy_user_config_.find(short_user);
if (proxy_user != authorized_proxy_user_config_.end()) {
- for (const string& user: proxy_user->second) {
- if (user == "*" || user == do_as_user) return Status::OK();
+ boost::unordered_set<string> users = proxy_user->second;
+ if (users.find("*") != users.end() ||
+ users.find(do_as_user) != users.end()) {
+ return Status::OK();
}
}
+
+ if (authorized_proxy_group_config_.size() > 0) {
+ // Check if the groups of do_as_user are in the authorized proxy groups.
+ AuthorizedProxyMap::const_iterator proxy_group =
+ authorized_proxy_group_config_.find(short_user);
+ if (proxy_group != authorized_proxy_group_config_.end()) {
+ boost::unordered_set<string> groups = proxy_group->second;
+ if (groups.find("*") != groups.end()) return Status::OK();
+
+ TGetHadoopGroupsRequest req;
+ req.__set_user(do_as_user);
+ TGetHadoopGroupsResponse res;
+ int64_t start = MonotonicMillis();
+ Status status = exec_env_->frontend()->GetHadoopGroups(req, &res);
+ VLOG_QUERY << "Getting Hadoop groups for user: " << short_user << " took " <<
+ (PrettyPrinter::Print(MonotonicMillis() - start, TUnit::TIME_MS));
+ if (!status.ok()) {
+ LOG(ERROR) << "Error getting Hadoop groups for user: " << short_user << ": "
+ << status.GetDetail();
+ return status;
+ }
+
+ for (const string& do_as_group : res.groups) {
+ if (groups.find(do_as_group) != groups.end()) {
+ return Status::OK();
+ }
+ }
+ }
+ }
+
VLOG(1) << error_msg;
return Status::Expected(error_msg.str());
}
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index 26e438f..deca000 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -466,6 +466,7 @@ class ImpalaServer : public ImpalaServiceIf,
friend class ChildQuery;
friend class ImpalaHttpHandler;
friend struct SessionState;
+ friend class ImpalaServerTest;
boost::scoped_ptr<ImpalaHttpHandler> http_handler_;
@@ -813,6 +814,18 @@ class ImpalaServer : public ImpalaServiceIf,
/// Expire 'crs' and cancel it with status 'status'.
void ExpireQuery(ClientRequestState* crs, const Status& status);
+ typedef boost::unordered_map<std::string, boost::unordered_set<std::string>>
+ AuthorizedProxyMap;
+ /// Populates authorized proxy config into the given map.
+ /// For example:
+ /// - authorized_proxy_config: foo=abc,def;bar=ghi
+ /// - authorized_proxy_config_delimiter: ,
+ /// - authorized_proxy_map: {foo:[abc, def], bar=s[ghi]}
+ static Status PopulateAuthorizedProxyConfig(
+ const std::string& authorized_proxy_config,
+ const std::string& authorized_proxy_config_delimiter,
+ AuthorizedProxyMap* authorized_proxy_map);
+
/// Guards query_log_ and query_log_index_
boost::mutex query_log_lock_;
@@ -1013,12 +1026,14 @@ class ImpalaServer : public ImpalaServiceIf,
/// update. Updated with each catalog topic heartbeat from the statestore.
int64_t min_subscriber_catalog_topic_version_;
- /// Map of short usernames of authorized proxy users to the set of user(s) they are
+ /// Map of short usernames of authorized proxy users to the set of users they are
/// allowed to delegate to. Populated by parsing the --authorized_proxy_users_config
/// flag.
- typedef boost::unordered_map<std::string, boost::unordered_set<std::string>>
- ProxyUserMap;
- ProxyUserMap authorized_proxy_user_config_;
+ AuthorizedProxyMap authorized_proxy_user_config_;
+ /// Map of short usernames of authorized proxy users to the set of groups they are
+ /// allowed to delegate to. Populated by parsing the --authorized_proxy_groups_config
+ /// flag.
+ AuthorizedProxyMap authorized_proxy_group_config_;
/// Guards queries_by_timestamp_. See "Locking" in the class comment for lock
/// acquisition order.
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 02e1ed8..acbdc6f 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -45,6 +45,8 @@ DECLARE_string(authorization_policy_file);
DECLARE_string(authorization_policy_provider_class);
DECLARE_string(authorized_proxy_user_config);
DECLARE_string(authorized_proxy_user_config_delimiter);
+DECLARE_string(authorized_proxy_group_config);
+DECLARE_string(authorized_proxy_group_config_delimiter);
DECLARE_string(kudu_master_hosts);
DECLARE_string(reserved_words_version);
DECLARE_string(sentry_config);
@@ -90,6 +92,7 @@ Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
}
cfg.__set_max_filter_error_rate(FLAGS_max_filter_error_rate);
cfg.__set_min_buffer_size(FLAGS_min_buffer_size);
+ cfg.__set_authorized_proxy_group_config(FLAGS_authorized_proxy_group_config);
RETURN_IF_ERROR(SerializeThriftMsg(jni_env, &cfg, cfg_bytes));
return Status::OK();
}
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index c98f50a..f645e3c 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -75,4 +75,6 @@ struct TBackendGflags {
24: required i64 min_buffer_size
25: required bool enable_orc_scanner
+
+ 26: required string authorized_proxy_group_config
}
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index f8ab05e..8d32b53 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -820,6 +820,16 @@ struct TGetAllHadoopConfigsResponse {
1: optional map<string, string> configs;
}
+struct TGetHadoopGroupsRequest {
+ // The user name to get the groups from.
+ 1: required string user
+}
+
+struct TGetHadoopGroupsResponse {
+ // The list of groups that the user belongs to.
+ 1: required list<string> groups
+}
+
// For creating a test descriptor table. The tuples and their memory layout are computed
// in the FE.
struct TBuildTestDescriptorTableParams {
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index a94f46e..9dd3f94 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -83,6 +83,10 @@ public class BackendConfig {
public long getMinBufferSize() { return backendCfg_.min_buffer_size; }
+ public boolean isAuthorizedProxyGroupEnabled() {
+ return !Strings.isNullOrEmpty(backendCfg_.authorized_proxy_group_config);
+ }
+
// Inits the auth_to_local configuration in the static KerberosName class.
private static void initAuthToLocal() {
// If auth_to_local is enabled, we read the configuration hadoop.security.auth_to_local
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index ea3b358..a343182 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -19,14 +19,15 @@ package org.apache.impala.service;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -34,6 +35,11 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback;
+import org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMappingWithFallback;
+import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
+import org.apache.hadoop.security.ShellBasedUnixGroupsNetgroupMapping;
import org.apache.impala.analysis.DescriptorTable;
import org.apache.impala.analysis.ToSqlUtils;
import org.apache.impala.authorization.AuthorizationConfig;
@@ -69,6 +75,8 @@ import org.apache.impala.thrift.TGetFunctionsParams;
import org.apache.impala.thrift.TGetFunctionsResult;
import org.apache.impala.thrift.TGetHadoopConfigRequest;
import org.apache.impala.thrift.TGetHadoopConfigResponse;
+import org.apache.impala.thrift.TGetHadoopGroupsRequest;
+import org.apache.impala.thrift.TGetHadoopGroupsResponse;
import org.apache.impala.thrift.TGetTablesParams;
import org.apache.impala.thrift.TGetTablesResult;
import org.apache.impala.thrift.TLoadDataReq;
@@ -569,6 +577,7 @@ public class JniFrontend {
// Caching this saves ~50ms per call to getHadoopConfigAsHtml
private static final Configuration CONF = new Configuration();
+ private static final Groups GROUPS = Groups.getUserToGroupsMappingService(CONF);
/**
* Returns a string of all loaded Hadoop configuration parameters as a table of keys
@@ -608,14 +617,68 @@ public class JniFrontend {
}
/**
+ * Returns the list of Hadoop groups for the given user name.
+ */
+ public byte[] getHadoopGroups(byte[] serializedRequest) throws ImpalaException {
+ TGetHadoopGroupsRequest request = new TGetHadoopGroupsRequest();
+ JniUtil.deserializeThrift(protocolFactory_, request, serializedRequest);
+ TGetHadoopGroupsResponse result = new TGetHadoopGroupsResponse();
+ try {
+ result.setGroups(GROUPS.getGroups(request.getUser()));
+ } catch (IOException e) {
+ // HACK: https://issues.apache.org/jira/browse/HADOOP-15505
+ // There is no easy way to know if no groups found for a user
+ // other than reading the exception message.
+ if (e.getMessage().startsWith("No groups found for user")) {
+ result.setGroups(Collections.<String>emptyList());
+ } else {
+ LOG.error("Error getting Hadoop groups for user: " + request.getUser(), e);
+ throw new InternalException(e.getMessage());
+ }
+ }
+ TSerializer serializer = new TSerializer(protocolFactory_);
+ try {
+ return serializer.serialize(result);
+ } catch (TException e) {
+ throw new InternalException(e.getMessage());
+ }
+ }
+
+ /**
+ * Returns an error string describing configuration issue with the groups mapping
+ * provider implementation.
+ */
+ @VisibleForTesting
+ protected static String checkGroupsMappingProvider(Configuration conf) {
+ String provider = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING);
+ // Shell-based groups mapping providers fork a new process for each call.
+ // This can cause issues such as zombie processes, running out of file descriptors,
+ // etc.
+ if (ShellBasedUnixGroupsNetgroupMapping.class.getName().equals(provider)) {
+ return String.format("Hadoop groups mapping provider: %s is " +
+ "known to be problematic. Consider using: %s instead.",
+ provider, JniBasedUnixGroupsNetgroupMappingWithFallback.class.getName());
+ }
+ if (ShellBasedUnixGroupsMapping.class.getName().equals(provider)) {
+ return String.format("Hadoop groups mapping provider: %s is " +
+ "known to be problematic. Consider using: %s instead.",
+ provider, JniBasedUnixGroupsMappingWithFallback.class.getName());
+ }
+ return "";
+ }
+
+ /**
* Returns an error string describing all configuration issues. If no config issues are
* found, returns an empty string.
*/
- public String checkConfiguration() {
+ public String checkConfiguration() throws ImpalaException {
StringBuilder output = new StringBuilder();
output.append(checkLogFilePermission());
output.append(checkFileSystem(CONF));
output.append(checkShortCircuitRead(CONF));
+ if (BackendConfig.INSTANCE.isAuthorizedProxyGroupEnabled()) {
+ output.append(checkGroupsMappingProvider(CONF));
+ }
return output.toString();
}
diff --git a/fe/src/test/java/org/apache/impala/service/JniFrontendTest.java b/fe/src/test/java/org/apache/impala/service/JniFrontendTest.java
new file mode 100644
index 0000000..b4f8dd8
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/service/JniFrontendTest.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback;
+import org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMappingWithFallback;
+import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
+import org.apache.hadoop.security.ShellBasedUnixGroupsNetgroupMapping;
+import org.apache.impala.common.ImpalaException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class JniFrontendTest {
+ @Test
+ public void testCheckGroupsMappingProvider() throws ImpalaException {
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ JniBasedUnixGroupsMappingWithFallback.class.getName());
+ assertTrue(JniFrontend.checkGroupsMappingProvider(conf).isEmpty());
+
+ conf = new Configuration();
+ conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ ShellBasedUnixGroupsMapping.class.getName());
+ assertEquals(JniFrontend.checkGroupsMappingProvider(conf),
+ String.format("Hadoop groups mapping provider: %s is known to be problematic. " +
+ "Consider using: %s instead.",
+ ShellBasedUnixGroupsMapping.class.getName(),
+ JniBasedUnixGroupsMappingWithFallback.class.getName()));
+
+ conf = new Configuration();
+ conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ ShellBasedUnixGroupsNetgroupMapping.class.getName());
+ assertEquals(JniFrontend.checkGroupsMappingProvider(conf),
+ String.format("Hadoop groups mapping provider: %s is known to be problematic. " +
+ "Consider using: %s instead.",
+ ShellBasedUnixGroupsNetgroupMapping.class.getName(),
+ JniBasedUnixGroupsNetgroupMappingWithFallback.class.getName()));
+ }
+}
\ No newline at end of file
diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py
index 0ae0c76..5b4bccd 100644
--- a/tests/authorization/test_authorization.py
+++ b/tests/authorization/test_authorization.py
@@ -22,6 +22,7 @@ import pytest
import shutil
import tempfile
import json
+import grp
from time import sleep, time
from getpass import getuser
from ImpalaService import ImpalaHiveServer2Service
@@ -35,6 +36,9 @@ from tests.util.filesystem_utils import WAREHOUSE
AUTH_POLICY_FILE = "%s/authz-policy.ini" % WAREHOUSE
+def get_groups():
+ return [grp.getgrgid(group).gr_name for group in os.getgroups()]
+
class TestAuthorization(CustomClusterTestSuite):
AUDIT_LOG_DIR = tempfile.mkdtemp(dir=os.getenv('LOG_DIR'))
@@ -174,10 +178,42 @@ class TestAuthorization(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--server_name=server1\
--authorization_policy_file=%s\
- --authorized_proxy_user_config=hue=%s\
+ --authorized_proxy_user_config=foo=bar;hue=%s\
--abort_on_failed_audit_event=false\
--audit_event_log_dir=%s" % (AUTH_POLICY_FILE, getuser(), AUDIT_LOG_DIR))
- def test_impersonation(self):
+ def test_user_impersonation(self):
+ """End-to-end user impersonation + authorization test"""
+ self.__test_impersonation()
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args("--server_name=server1\
+ --authorization_policy_file=%s\
+ --authorized_proxy_user_config=hue=bar\
+ --authorized_proxy_group_config=foo=bar;hue=%s\
+ --abort_on_failed_audit_event=false\
+ --audit_event_log_dir=%s" % (AUTH_POLICY_FILE,
+ ','.join(get_groups()),
+ AUDIT_LOG_DIR))
+ def test_group_impersonation(self):
+ """End-to-end group impersonation + authorization test"""
+ self.__test_impersonation()
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args("--server_name=server1\
+ --authorization_policy_file=%s\
+ --authorized_proxy_user_config=foo=bar\
+ --authorized_proxy_group_config=foo=bar\
+ --abort_on_failed_audit_event=false\
+ --audit_event_log_dir=%s" % (AUTH_POLICY_FILE, AUDIT_LOG_DIR))
+ def test_no_matching_user_and_group_impersonation(self):
+ open_session_req = TCLIService.TOpenSessionReq()
+ open_session_req.username = 'hue'
+ open_session_req.configuration = dict()
+ open_session_req.configuration['impala.doas.user'] = 'abc'
+ resp = self.hs2_client.OpenSession(open_session_req)
+ assert 'User \'hue\' is not authorized to delegate to \'abc\'' in str(resp)
+
+ def __test_impersonation(self):
"""End-to-end impersonation + authorization test. Expects authorization to be
configured before running this test"""
# TODO: To reuse the HS2 utility code from the TestHS2 test suite we need to import