You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ar...@apache.org on 2023/03/25 20:59:53 UTC
[impala] branch master updated: IMPALA-12015: enable healthz endpoint in admissiond webui
This is an automated email from the ASF dual-hosted git repository.
arawat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 00ebec388 IMPALA-12015: enable healthz endpoint in admissiond webui
00ebec388 is described below
commit 00ebec38886d1a951d8555a210719daeae4c88be
Author: Abhishek Rawat <ar...@cloudera.com>
AuthorDate: Wed Mar 22 21:16:19 2023 -0700
IMPALA-12015: enable healthz endpoint in admissiond webui
Exposed '/healthz' endpoint in admissiond's webserver and metrics
webserver. The handler for '/healthz' returns 'OK' if the statestore
subscriber service and KRPC service have started on admissiond.
Testing:
- Manually tested '/healthz' on both admissiond webserver and admissiond
metrics webserver
- Updated existing 'LdapWebserverTest' to also include test for
admissiond's metrics webserver
Change-Id: Iffb5ee7a00992d1bdba1deddf181ae38aa206140
Reviewed-on: http://gerrit.cloudera.org:8080/19647
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/scheduling/admission-control-service.h | 8 +++++
be/src/scheduling/admissiond-env.cc | 8 +++++
be/src/service/impala-http-handler.cc | 20 +++++++------
.../impala/customcluster/CustomClusterRunner.java | 34 +++++++++++++++++-----
.../impala/customcluster/LdapWebserverTest.java | 31 +++++++++++---------
5 files changed, 71 insertions(+), 30 deletions(-)
diff --git a/be/src/scheduling/admission-control-service.h b/be/src/scheduling/admission-control-service.h
index eac3c48e6..f9bb50012 100644
--- a/be/src/scheduling/admission-control-service.h
+++ b/be/src/scheduling/admission-control-service.h
@@ -74,8 +74,13 @@ class AdmissionControlService : public AdmissionControlServiceIf,
/// appear in 'current_backends'. Called in response to statestore updates.
void CancelQueriesOnFailedCoordinators(std::unordered_set<UniqueIdPB> current_backends);
+ /// Returns whether AdmissionControlService is healthy and is able to accept admission
+ /// related RPCs.
+ bool IsHealthy() { return service_started_.load(); }
+
private:
friend class ImpalaHttpHandler;
+ friend class AdmissiondEnv;
struct AdmissionState {
public:
@@ -154,6 +159,9 @@ class AdmissionControlService : public AdmissionControlServiceIf,
/// version to 'update_version' if 'update_version' is higher. Returns true if update
/// was successful.
bool CheckAndUpdateHeartbeat(const UniqueIdPB& coord_id, int64_t update_version);
+
+ /// Indicates whether the admission control service is ready.
+ std::atomic_bool service_started_{false};
};
} // namespace impala
diff --git a/be/src/scheduling/admissiond-env.cc b/be/src/scheduling/admissiond-env.cc
index 7c01cc916..27fbf8f99 100644
--- a/be/src/scheduling/admissiond-env.cc
+++ b/be/src/scheduling/admissiond-env.cc
@@ -87,6 +87,10 @@ Status AdmissiondEnv::Init() {
DaemonEnv::GetInstance()->metrics(), "mem-tracker.process");
http_handler_->RegisterHandlers(DaemonEnv::GetInstance()->webserver());
+ if (DaemonEnv::GetInstance()->metrics_webserver() != nullptr) {
+ http_handler_->RegisterHandlers(
+ DaemonEnv::GetInstance()->metrics_webserver(), /* metrics_only */ true);
+ }
IpAddr ip_address;
RETURN_IF_ERROR(HostnameToIpAddr(FLAGS_hostname, &ip_address));
@@ -126,6 +130,10 @@ Status AdmissiondEnv::StartServices() {
LOG(INFO) << "Starting KRPC service.";
RETURN_IF_ERROR(rpc_mgr_->StartServices());
+ // Mark service as started.
+ // Should be called only after the statestore subscriber service and KRPC service
+ // has started.
+ admission_control_svc_->service_started_ = true;
return Status::OK();
}
diff --git a/be/src/service/impala-http-handler.cc b/be/src/service/impala-http-handler.cc
index 1bdc2411d..6312fb603 100644
--- a/be/src/service/impala-http-handler.cc
+++ b/be/src/service/impala-http-handler.cc
@@ -105,6 +105,14 @@ ImpalaHttpHandler::ImpalaHttpHandler(ImpalaServer* server,
void ImpalaHttpHandler::RegisterHandlers(Webserver* webserver, bool metrics_only) {
DCHECK(webserver != NULL);
+ Webserver::RawUrlCallback healthz_callback =
+ [this](const auto& req, auto* data, auto* response) {
+ return this->HealthzHandler(req, data, response);
+ };
+ webserver->RegisterUrlCallback("/healthz", healthz_callback);
+
+ if (metrics_only) return;
+
if (is_admissiond_) {
// The admissiond only exposes a subset of endpoints that have info relevant to
// admission control.
@@ -119,14 +127,6 @@ void ImpalaHttpHandler::RegisterHandlers(Webserver* webserver, bool metrics_only
return;
}
- Webserver::RawUrlCallback healthz_callback =
- [this](const auto& req, auto* data, auto* response) {
- return this->HealthzHandler(req, data, response);
- };
- webserver->RegisterUrlCallback("/healthz", healthz_callback);
-
- if (metrics_only) return;
-
webserver->RegisterUrlCallback("/backends", "backends.tmpl",
MakeCallback(this, &ImpalaHttpHandler::BackendsHandler), true);
@@ -210,7 +210,9 @@ void ImpalaHttpHandler::RegisterHandlers(Webserver* webserver, bool metrics_only
void ImpalaHttpHandler::HealthzHandler(const Webserver::WebRequest& req,
std::stringstream* data, HttpStatusCode* response) {
- if (server_->IsHealthy()) {
+ if ((server_ != nullptr && server_->IsHealthy()) ||
+ (is_admissiond_ &&
+ AdmissiondEnv::GetInstance()->admission_control_service()->IsHealthy())) {
(*data) << "OK";
*response = HttpStatusCode::Ok;
return;
diff --git a/fe/src/test/java/org/apache/impala/customcluster/CustomClusterRunner.java b/fe/src/test/java/org/apache/impala/customcluster/CustomClusterRunner.java
index ecc1924f2..536d4dd15 100644
--- a/fe/src/test/java/org/apache/impala/customcluster/CustomClusterRunner.java
+++ b/fe/src/test/java/org/apache/impala/customcluster/CustomClusterRunner.java
@@ -54,31 +54,49 @@ class CustomClusterRunner {
*/
public static int StartImpalaCluster(String args, Map<String, String> env,
String startArgs) throws IOException, InterruptedException {
- return StartImpalaCluster(args, args, args, env, startArgs);
+ return StartImpalaCluster(args, args, args, "", env, startArgs);
}
public static int StartImpalaCluster(String impaladArgs, String catalogdArgs,
String statestoredArgs) throws IOException, InterruptedException {
return StartImpalaCluster(
- impaladArgs, catalogdArgs, statestoredArgs, new HashMap<String, String>(), "");
+ impaladArgs, catalogdArgs, statestoredArgs, "", new HashMap<String, String>(),
+ "");
}
public static int StartImpalaCluster(String impaladArgs, String catalogdArgs,
String statestoredArgs, String startArgs) throws IOException, InterruptedException {
- return StartImpalaCluster(impaladArgs, catalogdArgs, statestoredArgs,
+ return StartImpalaCluster(impaladArgs, catalogdArgs, statestoredArgs, "",
new HashMap<String, String>(), startArgs);
}
+ public static int StartImpalaCluster(String impaladArgs, String catalogdArgs,
+ String statestoredArgs, Map<String, String> env, String startArgs)
+ throws IOException, InterruptedException {
+ return StartImpalaCluster(impaladArgs, catalogdArgs, statestoredArgs, "",
+ env, startArgs);
+ }
+
/**
* Starts Impala, setting environment variables in 'env', and passing 'impalad_args',
- * 'catalogd_args', 'statestored_args', and 'startArgs' to start-impala-cluster.py.
+ * 'catalogd_args', 'statestored_args', 'admissiond_args' and 'startArgs' to
+ * start-impala-cluster.py.
*/
public static int StartImpalaCluster(String impaladArgs, String catalogdArgs,
- String statestoredArgs, Map<String, String> env, String startArgs)
+ String statestoredArgs, String admissiondArgs, Map<String, String> env,
+ String startArgs)
throws IOException, InterruptedException {
- ProcessBuilder pb = new ProcessBuilder(new String[] {"start-impala-cluster.py",
- "--impalad_args", impaladArgs, "--catalogd_args", catalogdArgs,
- "--state_store_args", statestoredArgs, startArgs});
+ ProcessBuilder pb;
+ if (!admissiondArgs.isEmpty()) {
+ pb = new ProcessBuilder(new String[] {"start-impala-cluster.py",
+ "--impalad_args", impaladArgs, "--catalogd_args", catalogdArgs,
+ "--state_store_args", statestoredArgs, "--enable_admission_service", "true",
+ "--admissiond_args", admissiondArgs, startArgs});
+ } else {
+ pb = new ProcessBuilder(new String[] {"start-impala-cluster.py",
+ "--impalad_args", impaladArgs, "--catalogd_args", catalogdArgs,
+ "--state_store_args", statestoredArgs, startArgs});
+ }
pb.redirectErrorStream(true);
Map<String, String> origEnv = pb.environment();
origEnv.putAll(env);
diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapWebserverTest.java b/fe/src/test/java/org/apache/impala/customcluster/LdapWebserverTest.java
index c17b3f73a..62ae0e9e6 100644
--- a/fe/src/test/java/org/apache/impala/customcluster/LdapWebserverTest.java
+++ b/fe/src/test/java/org/apache/impala/customcluster/LdapWebserverTest.java
@@ -69,7 +69,7 @@ public class LdapWebserverTest {
WebClient client_ = new WebClient(TEST_USER_1, TEST_PASSWORD_1);
public void setUp(String extraArgs, String startArgs, String catalogdArgs,
- String stateStoredArgs) throws Exception {
+ String stateStoredArgs, String admissiondArgs) throws Exception {
String uri =
String.format("ldap://localhost:%s", serverRule.getLdapServer().getPort());
String dn = "cn=#UID,ou=Users,dc=myorg,dc=com";
@@ -84,7 +84,7 @@ public class LdapWebserverTest {
catalogdArgs = catalogdArgs + " " + impalaArgs;
stateStoredArgs = stateStoredArgs + " " + impalaArgs;
int ret = CustomClusterRunner.StartImpalaCluster(impalaArgs, catalogdArgs,
- stateStoredArgs, env, startArgs);
+ stateStoredArgs, admissiondArgs, env, startArgs);
assertEquals(0, ret);
}
@@ -144,7 +144,7 @@ public class LdapWebserverTest {
@Test
public void testWebserver() throws Exception {
- setUp("", "", "", "");
+ setUp("", "", "", "", "");
// start-impala-cluster contacts the webui to confirm the impalads have started, so
// there will already be some successful auth attempts.
verifyMetrics(Range.atLeast(1L), zero, Range.atLeast(1L), zero);
@@ -178,7 +178,7 @@ public class LdapWebserverTest {
+ "--ldap_group_class_key=groupOfUniqueNames "
+ "--ldap_bind_dn=%s --ldap_bind_password_cmd='echo -n %s' ",
TEST_USER_GROUP, TEST_USER_1, TEST_USER_3, TEST_USER_DN_1, TEST_PASSWORD_1),
- "", "", "");
+ "", "", "", "");
// start-impala-cluster contacts the webui to confirm the impalads have started, so
// there will already be some successful auth attempts.
verifyMetrics(Range.atLeast(1L), zero, Range.atLeast(1L), zero);
@@ -214,9 +214,10 @@ public class LdapWebserverTest {
@Test
public void testMetricsWebserver() throws Exception {
// Use 'per_impalad_args' to turn the metrics webserver on only for the first impalad.
- setUp("", "--per_impalad_args=--metrics_webserver_port=25030 ",
+ setUp("", "--per_impalad_args=--metrics_webserver_port=25040 ",
"--metrics_webserver_port=25021 ",
- "--metrics_webserver_port=25011 ");
+ "--metrics_webserver_port=25011 ",
+ "--metrics_webserver_port=25031 ");
// Attempt to access the regular webserver without a username/password, should fail.
WebClient noUsername = new WebClient();
String result = noUsername.readContent("/");
@@ -228,9 +229,10 @@ public class LdapWebserverTest {
assertTrue(result, result.contains("Must authenticate with Basic authentication."));
// Attempt to access the metrics webserver without a username/password.
- WebClient noUsernameMetrics = new WebClient(25030);
+ WebClient noUsernameMetrics = new WebClient(25040);
WebClient catalogdMetrics = new WebClient(25021);
WebClient statestoredMetrics = new WebClient(25011);
+ WebClient admissiondMetrics = new WebClient(25031);
// Should succeed for the metrics endpoints.
for (String endpoint :
new String[] {"/metrics", "/jsonmetrics", "/metrics_prometheus", "/healthz"}) {
@@ -243,6 +245,9 @@ public class LdapWebserverTest {
result = statestoredMetrics.readContent(endpoint);
assertFalse(
result, result.contains("Must authenticate with Basic authentication."));
+ result = admissiondMetrics.readContent(endpoint);
+ assertFalse(
+ result, result.contains("Must authenticate with Basic authentication."));
}
for (String endpoint : new String[] {"/varz", "/backends"}) {
@@ -260,7 +265,7 @@ public class LdapWebserverTest {
String strictLocalhostArgs = "--trusted_domain_strict_localhost=" +
String.valueOf(strictLocalhost);
setUp("--trusted_domain=localhost --trusted_domain_use_xff_header=true " +
- strictLocalhostArgs, "", "", "");
+ strictLocalhostArgs, "", "", "", "");
// Case 1: Authenticate as 'Test1Ldap' with the right password '12345'
attemptConnection("Basic VGVzdDFMZGFwOjEyMzQ1", "127.0.0.1", false);
@@ -317,7 +322,7 @@ public class LdapWebserverTest {
@Test
public void testWebserverTrustedAuthHeader() throws Exception {
- setUp("--trusted_auth_header=X-Trusted-Proxy-Auth-Header", "", "", "");
+ setUp("--trusted_auth_header=X-Trusted-Proxy-Auth-Header", "", "", "", "");
// Case 1: Authenticate as 'Test1Ldap' with the right password '12345'.
attemptConnection("Basic VGVzdDFMZGFwOjEyMzQ1", null, true);
@@ -357,7 +362,7 @@ public class LdapWebserverTest {
"--jwt_token_auth=true --jwt_validate_signature=true --jwks_file_path=%s "
+ "--jwt_allow_without_tls=true",
jwksFilename),
- "", "", "");
+ "", "", "", "");
// Case 1: Authenticate with valid JWT Token in HTTP header.
String jwtToken =
@@ -389,7 +394,7 @@ public class LdapWebserverTest {
*/
@Test
public void testDisplaySrcUsernameInQueryCause() throws Exception {
- setUp("", "", "", "");
+ setUp("", "", "", "", "");
// Create client
THttpClient transport = new THttpClient("http://localhost:28000");
Map<String, String> headers = new HashMap<String, String>();
@@ -439,7 +444,7 @@ public class LdapWebserverTest {
*/
@Test
public void testSetGLogLevel() throws Exception {
- setUp("", "", "", "");
+ setUp("", "", "", "", "");
// Validate defaults
JSONObject json = client_.jsonGet("/log_level?json");
assertEquals("1", json.get("glog_level"));
@@ -514,7 +519,7 @@ public class LdapWebserverTest {
*/
@Test
public void testSetJavaLogLevel() throws Exception {
- setUp("", "", "", "");
+ setUp("", "", "", "", "");
// Validate defaults
JSONObject json = client_.jsonGet("/log_level?json");
assertEquals("org.apache.impala : DEBUG\n", json.get("get_java_loglevel_result"));