You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2020/04/02 21:46:05 UTC
[impala] branch master updated: IMPALA-9537: Add LDAP auth to the
webui
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 81b5f82 IMPALA-9537: Add LDAP auth to the webui
81b5f82 is described below
commit 81b5f823aa715d47f939191f2370e32ef9983413
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
AuthorDate: Fri Mar 20 11:19:39 2020 -0700
IMPALA-9537: Add LDAP auth to the webui
This patch adds a startup flag --webserver_require_ldap, which if set
to true secures the debug webserver with LDAP username/password
authentication.
It also adds the flags --webserver_ldap_group_filter and
--webserver_ldap_user_filter, which allow users to restrict access to
the webserver separately from the restrictions on other endpoints
(i.e. --ldap_group_filter and --ldap_user_filter), for example to
enable webserver access only for admin users.
Testing:
- Added a FE test that runs a custom cluster with ldap webserver auth
enabled and verifies it works as expected, with and without filters
specified.
Change-Id: I6e92481929f2f06898b8496233ab4134792c9f10
Reviewed-on: http://gerrit.cloudera.org:8080/15538
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/webserver.cc | 177 ++++++++++++++++-----
be/src/util/webserver.h | 17 ++
common/thrift/metrics.json | 26 ++-
.../impala/customcluster/CustomClusterRunner.java | 26 ++-
.../impala/customcluster/LdapImpalaShellTest.java | 55 +++----
.../impala/customcluster/LdapWebserverTest.java | 161 +++++++++++++++++++
.../java/org/apache/impala/testutil/LdapUtil.java | 36 +++++
.../test/java/org/apache/impala/util/Metrics.java | 71 ++++++---
tests/common/impala_service.py | 10 +-
9 files changed, 479 insertions(+), 100 deletions(-)
diff --git a/be/src/util/webserver.cc b/be/src/util/webserver.cc
index 6a9183b..cbf2874 100644
--- a/be/src/util/webserver.cc
+++ b/be/src/util/webserver.cc
@@ -34,12 +34,14 @@
#include "common/logging.h"
#include "gutil/endian.h"
-#include "gutil/strings/substitute.h"
+#include "gutil/strings/escaping.h"
#include "gutil/strings/strip.h"
+#include "gutil/strings/substitute.h"
+#include "kudu/security/gssapi.h"
#include "kudu/util/env.h"
#include "kudu/util/logging.h"
#include "kudu/util/net/sockaddr.h"
-#include "kudu/security/gssapi.h"
+#include "rpc/authentication.h"
#include "rpc/cookie-util.h"
#include "rpc/thrift-util.h"
#include "runtime/exec-env.h"
@@ -114,11 +116,23 @@ DEFINE_int32(webserver_max_post_length_bytes, 1024 * 1024,
"the embedded web server.");
DEFINE_bool(webserver_require_spnego, false,
- "Require connections to the web server to authenticate via Kerberos "
- "using SPNEGO.");
-
+ "Require connections to the web server to authenticate via Kerberos using SPNEGO. "
+ "Cannot be used with --webserver_require_ldap.");
+
+DEFINE_bool(webserver_require_ldap, false,
+ "Require connections to the web server to authenticate via LDAP using HTTP Basic "
+ "authentication. Cannot be used with --webserver_require_spnego.");
+DEFINE_string(webserver_ldap_user_filter, "",
+ "Comma separated list of usernames. If specified, users must be on this list for "
+ "LDAP athentication to the webserver to succeed.");
+DEFINE_string(webserver_ldap_group_filter, "",
+ "Comma separated list of groups. If specified, users must belong to one of these "
+ "groups for LDAP authentication to the webserver to succeed.");
+
+DECLARE_bool(enable_ldap_auth);
DECLARE_string(hostname);
DECLARE_bool(is_coordinator);
+DECLARE_bool(ldap_passwords_in_clear_ok);
DECLARE_int64(max_cookie_lifetime_s);
DECLARE_string(ssl_minimum_version);
DECLARE_string(ssl_cipher_list);
@@ -259,12 +273,18 @@ Webserver::Webserver(const string& interface, const int port, MetricGroup* metri
metrics->AddCounter("impala.webserver.total-negotiate-auth-success", 0);
total_negotiate_auth_failure_ =
metrics->AddCounter("impala.webserver.total-negotiate-auth-failure", 0);
- if (use_cookies_) {
- total_cookie_auth_success_ =
- metrics->AddCounter("impala.webserver.total-cookie-auth-success", 0);
- total_cookie_auth_failure_ =
- metrics->AddCounter("impala.webserver.total-cookie-auth-failure", 0);
- }
+ }
+ if (FLAGS_webserver_require_ldap) {
+ total_basic_auth_success_ =
+ metrics->AddCounter("impala.webserver.total-basic-auth-success", 0);
+ total_basic_auth_failure_ =
+ metrics->AddCounter("impala.webserver.total-basic-auth-failure", 0);
+ }
+ if (use_cookies_ && (FLAGS_webserver_require_spnego || FLAGS_webserver_require_ldap)) {
+ total_cookie_auth_success_ =
+ metrics->AddCounter("impala.webserver.total-cookie-auth-success", 0);
+ total_cookie_auth_failure_ =
+ metrics->AddCounter("impala.webserver.total-cookie-auth-failure", 0);
}
}
@@ -376,6 +396,11 @@ Status Webserver::Start() {
options.push_back(FLAGS_webserver_password_file.c_str());
}
+ if (FLAGS_webserver_require_spnego && FLAGS_webserver_require_ldap) {
+ return Status("Securing the web server with both Kerberos and LDAP is not "
+ "currently supported.");
+ }
+
if (FLAGS_webserver_require_spnego) {
// If Kerberos has been configured, security::InitKerberosForServer() will
// already have been called, ensuring that the keytab path has been
@@ -389,6 +414,21 @@ Status Webserver::Start() {
}
}
+ if (FLAGS_webserver_require_ldap) {
+ if (!FLAGS_enable_ldap_auth) {
+ return Status("Unable to secure web server with LDAP: LDAP authentication must be "
+ "configured for this daemon.");
+ }
+ if (!IsSecure() && !FLAGS_ldap_passwords_in_clear_ok) {
+ return Status("Unable to secure web server with LDAP: must either enable SSL or "
+ "set --ldap_passwords_in_clear_ok=true");
+ }
+
+ ldap_.reset(new ImpalaLdap());
+ RETURN_IF_ERROR(
+ ldap_->Init(FLAGS_webserver_ldap_user_filter, FLAGS_webserver_ldap_group_filter));
+ }
+
options.push_back("listening_ports");
options.push_back(listening_str.c_str());
@@ -523,50 +563,52 @@ sq_callback_result_t Webserver::BeginRequestCallback(struct sq_connection* conne
}
vector<string> response_headers;
- if (FLAGS_webserver_require_spnego){
- bool authenticated = false;
- // Try authenticating with a cookie first, if enabled.
- if (use_cookies_) {
- const char* cookie_header = sq_get_header(connection, "Cookie");
- string username;
- if (cookie_header != nullptr) {
- Status cookie_status = AuthenticateCookie(hash_, cookie_header, &username);
- if (cookie_status.ok()) {
- authenticated = true;
- request_info->remote_user = strdup(username.c_str());
- total_cookie_auth_success_->Increment(1);
- } else {
- LOG(INFO) << "Invalid cookie provided: " << cookie_header << ": "
- << cookie_status.GetDetail();
- response_headers.push_back(Substitute("Set-Cookie: $0", GetDeleteCookie()));
- total_cookie_auth_failure_->Increment(1);
- }
+ bool authenticated = !FLAGS_webserver_require_spnego && !FLAGS_webserver_require_ldap;
+ // Try authenticating with a cookie first, if enabled.
+ if (!authenticated && use_cookies_) {
+ const char* cookie_header = sq_get_header(connection, "Cookie");
+ string username;
+ if (cookie_header != nullptr) {
+ Status cookie_status = AuthenticateCookie(hash_, cookie_header, &username);
+ if (cookie_status.ok()) {
+ authenticated = true;
+ request_info->remote_user = strdup(username.c_str());
+ total_cookie_auth_success_->Increment(1);
+ } else {
+ LOG(INFO) << "Invalid cookie provided: " << cookie_header << ": "
+ << cookie_status.GetDetail();
+ response_headers.push_back(Substitute("Set-Cookie: $0", GetDeleteCookie()));
+ total_cookie_auth_failure_->Increment(1);
}
}
+ }
- if (!authenticated) {
+ if (!authenticated) {
+ if (FLAGS_webserver_require_spnego) {
sq_callback_result_t spnego_result =
HandleSpnego(connection, request_info, &response_headers);
if (spnego_result == SQ_CONTINUE_HANDLING) {
// Spnego negotiation was successful.
- if (use_cookies_) {
- // If cookie auth failed above and we generated a 'delete cookie' header,
- // remove it.
- auto eq = [](const string& header) {
- return header.rfind("Set-Cookie", 0) == 0;
- };
- auto it = find_if(response_headers.begin(), response_headers.end(), eq);
- if (it != response_headers.end()) {
- response_headers.erase(it);
- }
- // Generate a cookie to return.
- response_headers.push_back(Substitute(
- "Set-Cookie: $0", GenerateCookie(request_info->remote_user, hash_)));
- }
+ AddCookie(request_info, &response_headers);
} else {
// Spnego negotiation is incomplete or failed, stop processing the request.
return spnego_result;
}
+ } else {
+ DCHECK(FLAGS_webserver_require_ldap);
+ Status basic_status = HandleBasic(connection, request_info, &response_headers);
+ if (basic_status.ok()) {
+ // Basic auth was successful.
+ total_basic_auth_success_->Increment(1);
+ AddCookie(request_info, &response_headers);
+ } else {
+ total_basic_auth_failure_->Increment(1);
+ LOG(ERROR) << "Failed to authenticate: " << basic_status.GetDetail();
+ response_headers.push_back("WWW-Authenticate: Basic");
+ SendResponse(connection, "401 Authentication Required", "text/plain",
+ "Must authenticate with Basic authentication.", response_headers);
+ return SQ_HANDLED_OK;
+ }
}
}
@@ -701,6 +743,53 @@ sq_callback_result_t Webserver::HandleSpnego(struct sq_connection* connection,
return SQ_CONTINUE_HANDLING;
}
+Status Webserver::HandleBasic(struct sq_connection* connection,
+ struct sq_request_info* request_info, vector<string>* response_headers) {
+ const char* authz_header = sq_get_header(connection, "Authorization");
+ if (!authz_header) {
+ return Status::Expected("No Authorization header provided.");
+ }
+
+ string base64;
+ if (!TryStripPrefixString(authz_header, "Basic ", &base64)) {
+ return Status::Expected("No Basic authentication provided.");
+ }
+
+ string decoded;
+ if (!Base64Unescape(base64, &decoded)) {
+ return Status::Expected("Failed to decode base64 basic authentication token.");
+ }
+
+ std::size_t colon = decoded.find(':');
+ if (colon == std::string::npos) {
+ return Status::Expected("Invalid basic authentication token format, missing ':'.");
+ }
+ string username = decoded.substr(0, colon);
+ string password = decoded.substr(colon + 1);
+ bool ret = ldap_->LdapCheckPass(username.c_str(), password.c_str(), password.length());
+ if (ret) {
+ request_info->remote_user = strdup(username.c_str());
+ return Status::OK();
+ }
+
+ return Status::Expected("Failed to authenticate to LDAP.");
+}
+
+void Webserver::AddCookie(
+ struct sq_request_info* request_info, vector<string>* response_headers) {
+ if (use_cookies_) {
+ // If cookie auth failed and we generated a 'delete cookie' header, remove it.
+ auto eq = [](const string& header) { return header.rfind("Set-Cookie", 0) == 0; };
+ auto it = find_if(response_headers->begin(), response_headers->end(), eq);
+ if (it != response_headers->end()) {
+ response_headers->erase(it);
+ }
+ // Generate a cookie to return.
+ response_headers->push_back(
+ Substitute("Set-Cookie: $0", GenerateCookie(request_info->remote_user, hash_)));
+ }
+}
+
void Webserver::RenderUrlWithTemplate(const struct sq_connection* connection,
const WebRequest& req, const UrlHandler& url_handler, stringstream* output,
ContentType* content_type) {
diff --git a/be/src/util/webserver.h b/be/src/util/webserver.h
index 24ebbcc..feed82b 100644
--- a/be/src/util/webserver.h
+++ b/be/src/util/webserver.h
@@ -26,6 +26,7 @@
#include "common/status.h"
#include "kudu/util/web_callback_registry.h"
#include "thirdparty/squeasel/squeasel.h"
+#include "util/ldap-util.h"
#include "util/metrics-fwd.h"
#include "util/network-util.h"
#include "util/openssl-util.h"
@@ -189,6 +190,14 @@ class Webserver {
sq_callback_result_t HandleSpnego(struct sq_connection* connection,
struct sq_request_info* request_info, std::vector<std::string>* response_headers);
+ // Handle Basic authentication for this request. Returns an error if authentication was
+ // unsuccessful.
+ Status HandleBasic(struct sq_connection* connection,
+ struct sq_request_info* request_info, std::vector<std::string>* response_headers);
+
+ // Adds a 'Set-Cookie' header to 'response_headers', if cookie support is enabled.
+ void AddCookie(struct sq_request_info* request_info, vector<string>* response_headers);
+
/// Renders URLs through the Mustache templating library.
/// - Default ContentType is HTML.
/// - Argument 'raw' renders the page with PLAIN ContentType.
@@ -241,11 +250,19 @@ class Webserver {
/// If true and SPNEGO is in use, cookies will be used for authentication.
bool use_cookies_;
+ /// Used to validate usernames/passwords If LDAP authentication is in use.
+ std::unique_ptr<ImpalaLdap> ldap_;
+
/// If 'FLAGS_webserver_require_spnego' is true, metrics for the number of successful
/// and failed Negotiate auth attempts.
IntCounter* total_negotiate_auth_success_ = nullptr;
IntCounter* total_negotiate_auth_failure_ = nullptr;
+ /// If 'FLAGS_webserver_require_ldap' is true, metrics for the number of successful
+ /// and failed Basic auth attempts.
+ IntCounter* total_basic_auth_success_ = nullptr;
+ IntCounter* total_basic_auth_failure_ = nullptr;
+
/// If 'use_cookies_' is true, metrics for the number of successful and failed cookie
/// auth attempts.
IntCounter* total_cookie_auth_success_ = nullptr;
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index 99e2a78..e4be01a 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -2695,7 +2695,31 @@
"kind": "COUNTER",
"key": "impala.webserver.total-negotiate-auth-failure"
},
- {
+ {
+ "description": "The number of HTTP connection requests to this daemon's webserver that were successfully authenticated with LDAP",
+ "contexts": [
+ "IMPALAD",
+ "CATALOGSERVER",
+ "STATESTORE"
+ ],
+ "label": "Webserver HTTP Connection LDAP Auth Success",
+ "units": "NONE",
+ "kind": "COUNTER",
+ "key": "impala.webserver.total-basic-auth-success"
+ },
+ {
+ "description": "The number of HTTP connection requests to this daemon's webserver that attempted to authenticate with LDAP but were unsuccessful.",
+ "contexts": [
+ "IMPALAD",
+ "CATALOGSERVER",
+ "STATESTORE"
+ ],
+ "label": "Webserver HTTP Connection LDAP Auth Failure",
+ "units": "NONE",
+ "kind": "COUNTER",
+ "key": "impala.webserver.total-basic-auth-failure"
+ },
+ {
"description": "The number of HTTP connection requests to this daemon's webserver that were successfully authenticated using a cookie.",
"contexts": [
"IMPALAD",
diff --git a/fe/src/test/java/org/apache/impala/customcluster/CustomClusterRunner.java b/fe/src/test/java/org/apache/impala/customcluster/CustomClusterRunner.java
index 7081a9d..8e58394 100644
--- a/fe/src/test/java/org/apache/impala/customcluster/CustomClusterRunner.java
+++ b/fe/src/test/java/org/apache/impala/customcluster/CustomClusterRunner.java
@@ -18,6 +18,11 @@
package org.apache.impala.customcluster;
import java.io.IOException;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
/**
* Runs an Impala cluster with custom flags.
@@ -27,18 +32,33 @@ import java.io.IOException;
* not be used outside of this package.
*/
class CustomClusterRunner {
+ private static final Logger LOG = Logger.getLogger(CustomClusterRunner.class);
+
public static int StartImpalaCluster() throws IOException, InterruptedException {
return StartImpalaCluster("");
}
+ public static int StartImpalaCluster(String args)
+ throws IOException, InterruptedException {
+ return StartImpalaCluster(args, new HashMap<String, String>());
+ }
+
/**
- * Starts Impala and passes 'args' to the impalads, catalogd, and statestored.
+ * Starts Impala, setting environment variables in 'env', and passes 'args' to the
+ * impalads, catalogd, and statestored.
*/
- public static int StartImpalaCluster(String args)
+ public static int StartImpalaCluster(String args, Map<String, String> env)
throws IOException, InterruptedException {
- Process p = Runtime.getRuntime().exec(new String[] {"start-impala-cluster.py",
+ ProcessBuilder pb = new ProcessBuilder(new String[] {"start-impala-cluster.py",
"--impalad_args", args, "--catalogd_args", args, "--state_store_args", args});
+ pb.redirectErrorStream(true);
+ Map<String, String> origEnv = pb.environment();
+ origEnv.putAll(env);
+ Process p = pb.start();
p.waitFor();
+ // Print out the output of the process, for debugging. We only need to print stdout,
+ // as stderr is redirected to it above.
+ LOG.info(IOUtils.toString(p.getInputStream()));
return p.exitValue();
}
}
diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java b/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java
index 070aa33..1687130 100644
--- a/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java
+++ b/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java
@@ -17,6 +17,7 @@
package org.apache.impala.customcluster;
+import static org.apache.impala.testutil.LdapUtil.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -53,19 +54,7 @@ public class LdapImpalaShellTest {
@ClassRule
public static CreateLdapServerRule serverRule = new CreateLdapServerRule();
- // These correspond to the values in fe/src/test/resources/users.ldif
- private static final String testUser_ = "Test1Ldap";
- private static final String testPassword_ = "12345";
- private static final String testUser2_ = "Test2Ldap";
- private static final String testPassword2_ = "abcde";
- private static final String testUser3_ = "Test3Ldap";
- private static final String testPassword3_ = "67890";
- private static final String testUser4_ = "Test4Ldap";
- private static final String testPassword4_ = "fghij";
- // testUser_ and testUser2_ are members of this group.
- private static final String testUserGroup_ = "group1";
-
- // The cluster will be set up to allow testUser_ to act as a proxy for delegateUser_.
+ // The cluster will be set up to allow TEST_USER_1 to act as a proxy for delegateUser_.
// Includes a special character to test HTTP path encoding.
private static final String delegateUser_ = "proxyUser$";
@@ -110,9 +99,9 @@ public class LdapImpalaShellTest {
String query = "select logged_in_user()";
// Templated shell commands to test a simple 'show tables' command.
// 1. Valid username and password. Should succeed.
- String[] validCommand = {"impala-shell.sh", "", "--ldap",
- "--auth_creds_ok_in_clear", String.format("--user=%s", testUser_),
- String.format("--ldap_password_cmd=printf %s", testPassword_),
+ String[] validCommand = {"impala-shell.sh", "", "--ldap", "--auth_creds_ok_in_clear",
+ String.format("--user=%s", TEST_USER_1),
+ String.format("--ldap_password_cmd=printf %s", TEST_PASSWORD_1),
String.format("--query=%s", query)};
// 2. Invalid username password combination. Should fail.
String[] invalidCommand = {"impala-shell.sh", "", "--ldap",
@@ -131,7 +120,7 @@ public class LdapImpalaShellTest {
for (String protocol: protocolsToTest) {
protocol = String.format(protocolTemplate, protocol);
validCommand[1] = protocol;
- RunShellCommand.Run(validCommand, /*shouldSucceed*/ true, testUser_,
+ RunShellCommand.Run(validCommand, /*shouldSucceed*/ true, TEST_USER_1,
"Starting Impala Shell using LDAP-based authentication");
invalidCommand[1] = protocol;
RunShellCommand.Run(
@@ -157,8 +146,8 @@ public class LdapImpalaShellTest {
*/
@Test
public void testHttpImpersonation() throws Exception {
- setUp(
- String.format("--authorized_proxy_user_config=%s=%s", testUser_, delegateUser_));
+ setUp(String.format(
+ "--authorized_proxy_user_config=%s=%s", TEST_USER_1, delegateUser_));
// Ignore the test if python SSLContext support is not available.
Assume.assumeTrue(pythonSupportsSSLContext());
String invalidDelegateUser = "invalid-delegate-user";
@@ -167,25 +156,24 @@ public class LdapImpalaShellTest {
// Run with an invalid proxy user.
String[] command = buildCommand(
- query, "hs2-http", testUser2_, testPassword2_, "/?doAs=" + delegateUser_);
+ query, "hs2-http", TEST_USER_2, TEST_PASSWORD_2, "/?doAs=" + delegateUser_);
RunShellCommand.Run(command, /* shouldSucceed */ false, "",
- String.format(errTemplate, testUser2_, delegateUser_));
+ String.format(errTemplate, TEST_USER_2, delegateUser_));
// Run with a valid proxy user but invalid delegate user.
command = buildCommand(
- query, "hs2-http", testUser_, testPassword_, "/?doAs=" + invalidDelegateUser);
+ query, "hs2-http", TEST_USER_1, TEST_PASSWORD_1, "/?doAs=" + invalidDelegateUser);
RunShellCommand.Run(command, /* shouldSucceed */ false, "",
- String.format(errTemplate, testUser_, invalidDelegateUser));
+ String.format(errTemplate, TEST_USER_1, invalidDelegateUser));
// 'doAs' parameter that cannot be decoded.
- command = buildCommand(
- query, "hs2-http", testUser_, testPassword_, "/?doAs=%");
+ command = buildCommand(query, "hs2-http", TEST_USER_1, TEST_PASSWORD_1, "/?doAs=%");
RunShellCommand.Run(command, /* shouldSucceed */ false, "",
"Not connected to Impala");
// Successfully delegate.
command = buildCommand(
- query, "hs2-http", testUser_, testPassword_, "/?doAs=" + delegateUser_);
+ query, "hs2-http", TEST_USER_1, TEST_PASSWORD_1, "/?doAs=" + delegateUser_);
RunShellCommand.Run(command, /* shouldSucceed */ true, delegateUser_, "");
}
@@ -201,27 +189,30 @@ public class LdapImpalaShellTest {
+ "--ldap_group_dn_pattern=%s "
+ "--ldap_group_membership_key=uniqueMember "
+ "--ldap_group_class_key=groupOfUniqueNames",
- testUserGroup_, testUser_, testUser3_, groupDN));
+ TEST_USER_GROUP, TEST_USER_1, TEST_USER_3, groupDN));
String query = "select logged_in_user()";
// Run with user that passes the group filter but not the user filter, should fail.
String[] command =
- buildCommand(query, "hs2-http", testUser2_, testPassword2_, "/cliservice");
+ buildCommand(query, "hs2-http", TEST_USER_2, TEST_PASSWORD_2, "/cliservice");
RunShellCommand.Run(
command, /* shouldSucceed */ false, "", "Not connected to Impala");
// Run with user that passes the user filter but not the group filter, should fail.
- command = buildCommand(query, "hs2-http", testUser3_, testPassword3_, "/cliservice");
+ command =
+ buildCommand(query, "hs2-http", TEST_USER_3, TEST_PASSWORD_3, "/cliservice");
RunShellCommand.Run(
command, /* shouldSucceed */ false, "", "Not connected to Impala");
// Run with user that doesn't pass either filter, should fail.
- command = buildCommand(query, "hs2-http", testUser4_, testPassword4_, "/cliservice");
+ command =
+ buildCommand(query, "hs2-http", TEST_USER_4, TEST_PASSWORD_4, "/cliservice");
RunShellCommand.Run(
command, /* shouldSucceed */ false, "", "Not connected to Impala");
// Run with user that passes both filters, should succeed.
- command = buildCommand(query, "hs2-http", testUser_, testPassword_, "/cliservice");
- RunShellCommand.Run(command, /* shouldSucceed */ true, testUser_, "");
+ command =
+ buildCommand(query, "hs2-http", TEST_USER_1, TEST_PASSWORD_1, "/cliservice");
+ RunShellCommand.Run(command, /* shouldSucceed */ true, TEST_USER_1, "");
}
}
diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapWebserverTest.java b/fe/src/test/java/org/apache/impala/customcluster/LdapWebserverTest.java
new file mode 100644
index 0000000..e23189b
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/customcluster/LdapWebserverTest.java
@@ -0,0 +1,161 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.customcluster;
+
+import static org.apache.impala.testutil.LdapUtil.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.Range;
+import org.apache.directory.server.core.annotations.CreateDS;
+import org.apache.directory.server.core.annotations.CreatePartition;
+import org.apache.directory.server.annotations.CreateLdapServer;
+import org.apache.directory.server.annotations.CreateTransport;
+import org.apache.directory.server.core.annotations.ApplyLdifFiles;
+import org.apache.directory.server.core.integ.CreateLdapServerRule;
+import org.apache.impala.util.Metrics;
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+@CreateDS(name = "myDS",
+ partitions = { @CreatePartition(name = "test", suffix = "dc=myorg,dc=com") })
+@CreateLdapServer(
+ transports = { @CreateTransport(protocol = "LDAP", address = "localhost") })
+@ApplyLdifFiles({"users.ldif"})
+public class LdapWebserverTest {
+ private static final Logger LOG = Logger.getLogger(LdapWebserverTest.class);
+ @ClassRule
+ public static CreateLdapServerRule serverRule = new CreateLdapServerRule();
+
+ private static final Range<Long> zero = Range.closed(0L, 0L);
+
+ Metrics metrics_ = new Metrics(TEST_USER_1, TEST_PASSWORD_1);
+
+ public void setUp(String extraArgs) throws Exception {
+ String uri =
+ String.format("ldap://localhost:%s", serverRule.getLdapServer().getPort());
+ String dn = "cn=#UID,ou=Users,dc=myorg,dc=com";
+ String impalaArgs = String.format("--enable_ldap_auth --ldap_uri='%s' "
+ + "--ldap_bind_pattern='%s' --ldap_passwords_in_clear_ok "
+ + "--webserver_require_ldap=true %s",
+ uri, dn, extraArgs);
+ Map<String, String> env = new HashMap<>();
+ env.put("IMPALA_WEBSERVER_USERNAME", TEST_USER_1);
+ env.put("IMPALA_WEBSERVER_PASSWORD", TEST_PASSWORD_1);
+ int ret = CustomClusterRunner.StartImpalaCluster(impalaArgs, env);
+ assertEquals(ret, 0);
+ }
+
+ @After
+ public void cleanUp() throws IOException {
+ metrics_.Close();
+ }
+
+ private void verifyMetrics(Range<Long> expectedBasicSuccess,
+ Range<Long> expectedBasicFailure, Range<Long> expectedCookieSuccess,
+ Range<Long> expectedCookieFailure) throws Exception {
+ long actualBasicSuccess =
+ (long) metrics_.getMetric("impala.webserver.total-basic-auth-success");
+ assertTrue("Expected: " + expectedBasicSuccess + ", Actual: " + actualBasicSuccess,
+ expectedBasicSuccess.contains(actualBasicSuccess));
+ long actualBasicFailure =
+ (long) metrics_.getMetric("impala.webserver.total-basic-auth-failure");
+ assertTrue("Expected: " + expectedBasicFailure + ", Actual: " + actualBasicFailure,
+ expectedBasicFailure.contains(actualBasicFailure));
+
+ long actualCookieSuccess =
+ (long) metrics_.getMetric("impala.webserver.total-cookie-auth-success");
+ assertTrue("Expected: " + expectedCookieSuccess + ", Actual: " + actualCookieSuccess,
+ expectedCookieSuccess.contains(actualCookieSuccess));
+ long actualCookieFailure =
+ (long) metrics_.getMetric("impala.webserver.total-cookie-auth-failure");
+ assertTrue("Expected: " + expectedCookieFailure + ", Actual: " + actualCookieFailure,
+ expectedCookieFailure.contains(actualCookieFailure));
+ }
+
+ @Test
+ public void testWebserver() throws Exception {
+ setUp("");
+ // start-impala-cluster contacts the webui to confirm the impalads have started, so
+ // there will already be some successful auth attempts.
+ verifyMetrics(Range.atLeast(1L), zero, Range.atLeast(1L), zero);
+
+ // Attempt to access the webserver without a username/password.
+ Metrics noUsername = new Metrics();
+ String result = noUsername.readContent("/");
+ assertTrue(result, result.contains("Must authenticate with Basic authentication."));
+ // Check that there is one unsuccessful auth attempt.
+ verifyMetrics(Range.atLeast(1L), Range.closed(1L, 1L), Range.atLeast(1L), zero);
+
+ // Attempt to access the webserver with invalid username/password.
+ Metrics invalidUserPass = new Metrics("invalid", "invalid");
+ result = invalidUserPass.readContent("/");
+ assertTrue(result, result.contains("Must authenticate with Basic authentication."));
+ // Check that there is now two unsuccessful auth attempts.
+ verifyMetrics(Range.atLeast(1L), Range.closed(2L, 2L), Range.atLeast(1L), zero);
+ }
+
+ /**
+ * Tests the webserver specific LDAP user and group filter configs.
+ */
+ @Test
+ public void testWebserverFilters() throws Exception {
+ // Set up the filters. Note that we don't use the optional '%s' in the group dn
+ // pattern or list multiple groups, these features are covered by LdapImpalaShellTest.
+ setUp(String.format("--webserver_ldap_group_filter=%s "
+ + "--webserver_ldap_user_filter=%s,%s "
+ + "--ldap_group_dn_pattern=ou=Groups,dc=myorg,dc=com "
+ + "--ldap_group_membership_key=uniqueMember "
+ + "--ldap_group_class_key=groupOfUniqueNames",
+ TEST_USER_GROUP, TEST_USER_1, TEST_USER_3));
+ // start-impala-cluster contacts the webui to confirm the impalads have started, so
+ // there will already be some successful auth attempts.
+ verifyMetrics(Range.atLeast(1L), zero, Range.atLeast(1L), zero);
+
+ // Access the webserver with a user that passes the group filter but not the user
+ // filter, should fail.
+ Metrics metricsUser2 = new Metrics(TEST_USER_2, TEST_PASSWORD_2);
+ String result = metricsUser2.readContent("/");
+ assertTrue(result, result.contains("Must authenticate with Basic authentication."));
+ // Check that there is one unsuccessful auth attempt.
+ verifyMetrics(Range.atLeast(1L), Range.closed(1L, 1L), Range.atLeast(1L), zero);
+
+ // Access the webserver with a user that passes the user filter but not the group
+ // filter, should fail.
+ Metrics metricsUser3 = new Metrics(TEST_USER_3, TEST_PASSWORD_3);
+ result = metricsUser3.readContent("/");
+ assertTrue(result, result.contains("Must authenticate with Basic authentication."));
+ // Check that there is now two unsuccessful auth attempts.
+ verifyMetrics(Range.atLeast(1L), Range.closed(2L, 2L), Range.atLeast(1L), zero);
+
+ // Access the webserver with a user that doesn't pass either filter, should fail.
+ Metrics metricsUser4 = new Metrics(TEST_USER_4, TEST_PASSWORD_4);
+ result = metricsUser4.readContent("/");
+ assertTrue(result, result.contains("Must authenticate with Basic authentication."));
+ // Check that there is now three unsuccessful auth attempts.
+ verifyMetrics(Range.atLeast(1L), Range.closed(3L, 3L), Range.atLeast(1L), zero);
+ }
+}
diff --git a/fe/src/test/java/org/apache/impala/testutil/LdapUtil.java b/fe/src/test/java/org/apache/impala/testutil/LdapUtil.java
new file mode 100644
index 0000000..8f448d2
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/testutil/LdapUtil.java
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.testutil;
+
+public class LdapUtil {
+ // These correspond to the values in fe/src/test/resources/users.ldif
+ public static final String TEST_USER_1 = "Test1Ldap";
+ public static final String TEST_PASSWORD_1 = "12345";
+
+ public static final String TEST_USER_2 = "Test2Ldap";
+ public static final String TEST_PASSWORD_2 = "abcde";
+
+ public static final String TEST_USER_3 = "Test3Ldap";
+ public static final String TEST_PASSWORD_3 = "67890";
+
+ public static final String TEST_USER_4 = "Test4Ldap";
+ public static final String TEST_PASSWORD_4 = "fghij";
+
+ // TEST_USER_1 and TEST_USER_2 are members of this group.
+ public static final String TEST_USER_GROUP = "group1";
+}
diff --git a/fe/src/test/java/org/apache/impala/util/Metrics.java b/fe/src/test/java/org/apache/impala/util/Metrics.java
index 537f406..75a20f2 100644
--- a/fe/src/test/java/org/apache/impala/util/Metrics.java
+++ b/fe/src/test/java/org/apache/impala/util/Metrics.java
@@ -17,9 +17,22 @@
package org.apache.impala.util;
-import java.net.URL;
-import java.util.Scanner;
-
+import java.io.IOException;
+
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.AuthCache;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.HttpHost;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
@@ -27,19 +40,24 @@ import org.json.simple.parser.JSONParser;
* Utility class for retrieving metrics from the Impala webserver.
*/
public class Metrics {
- private final static String DEFAULT_WEBSERVER_BASE_URL = "http://localhost:25000/";
- private final static String JSON_METRICS = "jsonmetrics?json";
+ private final static String WEBSERVER_HOST = "localhost";
+ private final static int WEBSERVER_PORT = 25000;
+ private final static String JSON_METRICS = "/jsonmetrics?json";
- private String webserverBaseUrl;
+ private CloseableHttpClient httpClient_;
+ private String username_;
+ private String password_;
- public Metrics() {
- this.webserverBaseUrl = DEFAULT_WEBSERVER_BASE_URL;
- }
+ public Metrics() { this("", ""); }
- public Metrics(String webserverUrl) {
- this.webserverBaseUrl = webserverUrl;
+ public Metrics(String username, String password) {
+ this.username_ = username;
+ this.password_ = password;
+ httpClient_ = HttpClients.createDefault();
}
+ public void Close() throws IOException { httpClient_.close(); }
+
/**
* Returns the metric for the given metric id from the Impala web server.
* @param metricId identifier of the metric we want to retrieve
@@ -47,7 +65,7 @@ public class Metrics {
* The caller needs to cast it to the appropriate type, e.g. Long, String, etc.
*/
public Object getMetric(String metricId) throws Exception {
- String content = readContent(new URL(this.webserverBaseUrl + JSON_METRICS));
+ String content = readContent(JSON_METRICS);
if (content == null) return null;
JSONObject json = toJson(content);
@@ -56,14 +74,29 @@ public class Metrics {
return json.get(metricId);
}
- private static String readContent(URL url) throws Exception {
- String ret = null;
+ /**
+ * Retrieves the page at 'path' and returns its contents.
+ */
+ public String readContent(String path) throws IOException {
+ HttpHost targetHost = new HttpHost(WEBSERVER_HOST, WEBSERVER_PORT, "http");
+ HttpClientContext context = HttpClientContext.create();
+ if (!username_.equals("")) {
+ CredentialsProvider credsProvider = new BasicCredentialsProvider();
+ credsProvider.setCredentials(new AuthScope(WEBSERVER_HOST, WEBSERVER_PORT),
+ new UsernamePasswordCredentials(username_, password_));
+ AuthCache authCache = new BasicAuthCache();
+ authCache.put(targetHost, new BasicScheme());
+ context.setCredentialsProvider(credsProvider);
+ context.setAuthCache(authCache);
+ }
- try (Scanner scanner = new Scanner(url.openStream(), "UTF-8")) {
- scanner.useDelimiter("\\A");
- if (scanner.hasNext()) {
- ret = scanner.next();
- }
+ String ret = "";
+ HttpGet httpGet = new HttpGet(path);
+ CloseableHttpResponse response = httpClient_.execute(targetHost, httpGet, context);
+ try {
+ ret = EntityUtils.toString(response.getEntity());
+ } finally {
+ response.close();
}
return ret;
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index 9cb7f50..ee5a6e5 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -22,6 +22,7 @@
from collections import defaultdict
import json
import logging
+import os
import re
import requests
from time import sleep, time
@@ -39,6 +40,9 @@ import zlib
LOG = logging.getLogger('impala_service')
LOG.setLevel(level=logging.DEBUG)
+WEBSERVER_USERNAME = os.environ.get('IMPALA_WEBSERVER_USERNAME', None)
+WEBSERVER_PASSWORD = os.environ.get('IMPALA_WEBSERVER_PASSWORD', None)
+
# Base class for all Impala services
# TODO: Refactor the retry/timeout logic into a common place.
class BaseImpalaService(object):
@@ -51,6 +55,9 @@ class BaseImpalaService(object):
self.webserver_interface = hostname
self.webserver_port = webserver_port
self.webserver_certificate_file = webserver_certificate_file
+ self.webserver_username_password = None
+ if WEBSERVER_USERNAME is not None and WEBSERVER_PASSWORD is not None:
+ self.webserver_username_password = (WEBSERVER_USERNAME, WEBSERVER_PASSWORD)
def open_debug_webpage(self, page_name, timeout=10, interval=1):
start_time = time()
@@ -62,7 +69,8 @@ class BaseImpalaService(object):
protocol = "https"
url = "%s://%s:%d/%s" % \
(protocol, self.webserver_interface, int(self.webserver_port), page_name)
- return requests.get(url, verify=self.webserver_certificate_file)
+ return requests.get(url, verify=self.webserver_certificate_file,
+ auth=self.webserver_username_password)
except Exception as e:
LOG.info("Debug webpage not yet available: %s", str(e))
sleep(interval)