You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2020/09/09 00:13:52 UTC
[impala] branch master updated (fb6d96e -> 6aaea32)
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from fb6d96e IMPALA-9741: Support querying Iceberg table by impala
new 1e63722 IMPALA-10124 admission-controller-test fails with no such file or directory error
new fc51cd3 IMPALA-10052: Expose daemon health endpoint for statestore and catalog
new 6aaea32 IMPALA-10090 Pull newest code of native-toolchain before build it
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/catalog/catalog-server.cc | 19 +++++++++++++++++++
be/src/catalog/catalog-server.h | 14 ++++++++++++++
be/src/catalog/catalogd-main.cc | 1 +
be/src/scheduling/admission-controller-test.cc | 4 ++++
be/src/statestore/statestore.cc | 20 ++++++++++++++++++--
be/src/statestore/statestore.h | 10 ++++++++--
bin/bootstrap_system.sh | 3 +++
tests/webserver/test_web_pages.py | 6 ++++--
8 files changed, 71 insertions(+), 6 deletions(-)
[impala] 02/03: IMPALA-10052: Expose daemon health endpoint for
statestore and catalog
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit fc51cd3bc0fa5e57e7b15fc07dbff9ced5d089c6
Author: Bikramjeet Vig <bi...@gmail.com>
AuthorDate: Wed Aug 5 16:31:08 2020 -0700
IMPALA-10052: Expose daemon health endpoint for statestore and catalog
This change exposes the daemon health of statestored and catalogd via
an HTTP endpoint '/healthz'. If the server is healthy, this endpoint
will return HTTP code 200 (OK). If it is unhealthy, it will return
503 (Service Unavailable). This is consistent with the endpoint added
for impalads in IMPALA-8895.
Testing:
- Extended test in test_web_pages.py
Change-Id: I7714734df8e50dabbbebcb77a86a5a00bd13bf7c
Reviewed-on: http://gerrit.cloudera.org:8080/16295
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/catalog/catalog-server.cc | 19 +++++++++++++++++++
be/src/catalog/catalog-server.h | 14 ++++++++++++++
be/src/catalog/catalogd-main.cc | 1 +
be/src/statestore/statestore.cc | 20 ++++++++++++++++++--
be/src/statestore/statestore.h | 10 ++++++++--
tests/webserver/test_web_pages.py | 6 ++++--
6 files changed, 64 insertions(+), 6 deletions(-)
diff --git a/be/src/catalog/catalog-server.cc b/be/src/catalog/catalog-server.cc
index d38f241..b67172b 100644
--- a/be/src/catalog/catalog-server.cc
+++ b/be/src/catalog/catalog-server.cc
@@ -102,6 +102,7 @@ const string TABLE_METRICS_WEB_PAGE = "/table_metrics";
const string TABLE_METRICS_TEMPLATE = "table_metrics.tmpl";
const string EVENT_WEB_PAGE = "/events";
const string EVENT_METRICS_TEMPLATE = "events.tmpl";
+const string CATALOG_SERVICE_HEALTH_WEB_PAGE = "/healthz";
const int REFRESH_METRICS_INTERVAL_MS = 1000;
@@ -287,6 +288,11 @@ Status CatalogServer::Start() {
}
void CatalogServer::RegisterWebpages(Webserver* webserver) {
+ Webserver::RawUrlCallback healthz_callback =
+ [this](const auto& req, auto* data, auto* response) {
+ return this->HealthzHandler(req, data, response);
+ };
+ webserver->RegisterUrlCallback(CATALOG_SERVICE_HEALTH_WEB_PAGE, healthz_callback);
webserver->RegisterUrlCallback(CATALOG_WEB_PAGE, CATALOG_TEMPLATE,
[this](const auto& args, auto* doc) { this->CatalogUrlCallback(args, doc); }, true);
webserver->RegisterUrlCallback(CATALOG_OBJECT_WEB_PAGE, CATALOG_OBJECT_TEMPLATE,
@@ -740,3 +746,16 @@ bool CatalogServer::AddPendingTopicItem(std::string key, int64_t version,
Substitute(", compressed size=$0", item.value.size()) : string());
return true;
}
+
+void CatalogServer::MarkServiceAsStarted() { service_started_ = true; }
+
+void CatalogServer::HealthzHandler(
+ const Webserver::WebRequest& req, std::stringstream* data, HttpStatusCode* response) {
+ if (service_started_) {
+ (*data) << "OK";
+ *response = HttpStatusCode::Ok;
+ return;
+ }
+ *(data) << "Not Available";
+ *response = HttpStatusCode::ServiceUnavailable;
+}
diff --git a/be/src/catalog/catalog-server.h b/be/src/catalog/catalog-server.h
index dec22ea..5cb3a2f 100644
--- a/be/src/catalog/catalog-server.h
+++ b/be/src/catalog/catalog-server.h
@@ -27,11 +27,14 @@
#include "gen-cpp/Frontend_types.h"
#include "gen-cpp/Types_types.h"
#include "catalog/catalog.h"
+#include "kudu/util/web_callback_registry.h"
#include "statestore/statestore-subscriber.h"
#include "util/condition-variable.h"
#include "util/metrics-fwd.h"
#include "rapidjson/rapidjson.h"
+using kudu::HttpStatusCode;
+
namespace impala {
class StatestoreSubscriber;
@@ -78,7 +81,14 @@ class CatalogServer {
bool AddPendingTopicItem(std::string key, int64_t version, const uint8_t* item_data,
uint32_t size, bool deleted);
+ /// Mark service as started. Should be called only after the thrift server hosting this
+ /// service has started.
+ void MarkServiceAsStarted();
+
private:
+ /// Indicates whether the catalog service is ready.
+ std::atomic_bool service_started_{false};
+
/// Thrift API implementation which proxies requests onto this CatalogService.
boost::shared_ptr<CatalogServiceIf> thrift_iface_;
ThriftSerializer thrift_serializer_;
@@ -236,6 +246,10 @@ class CatalogServer {
// metastore event processor metrics and adds it to the document
void EventMetricsUrlCallback(
const Webserver::WebRequest& req, rapidjson::Document* document);
+
+ /// Raw callback to indicate whether the service is ready.
+ void HealthzHandler(const Webserver::WebRequest& req, std::stringstream* data,
+ HttpStatusCode* response);
};
}
diff --git a/be/src/catalog/catalogd-main.cc b/be/src/catalog/catalogd-main.cc
index d3bc474..ed03970 100644
--- a/be/src/catalog/catalogd-main.cc
+++ b/be/src/catalog/catalogd-main.cc
@@ -117,6 +117,7 @@ int CatalogdMain(int argc, char** argv) {
}
ABORT_IF_ERROR(builder.metrics(metrics.get()).Build(&server));
ABORT_IF_ERROR(server->Start());
+ catalog_server.MarkServiceAsStarted();
LOG(INFO) << "CatalogService started on port: " << FLAGS_catalog_service_port;
server->Join();
diff --git a/be/src/statestore/statestore.cc b/be/src/statestore/statestore.cc
index 9450294..165a003 100644
--- a/be/src/statestore/statestore.cc
+++ b/be/src/statestore/statestore.cc
@@ -457,7 +457,7 @@ Statestore::Statestore(MetricGroup* metrics)
}
Statestore::~Statestore() {
- CHECK(initialized_) << "Cannot shutdown Statestore once initialized.";
+ CHECK(service_started_) << "Cannot shutdown Statestore once initialized and started.";
}
Status Statestore::Init(int32_t state_store_port) {
@@ -486,12 +486,17 @@ Status Statestore::Init(int32_t state_store_port) {
RETURN_IF_ERROR(subscriber_heartbeat_threadpool_.Init());
RETURN_IF_ERROR(Thread::Create("statestore-heartbeat", "heartbeat-monitoring-thread",
&Statestore::MonitorSubscriberHeartbeat, this, &heartbeat_monitoring_thread_));
- initialized_ = true;
+ service_started_ = true;
return Status::OK();
}
void Statestore::RegisterWebpages(Webserver* webserver) {
+ Webserver::RawUrlCallback healthz_callback =
+ [this](const auto& req, auto* data, auto* response) {
+ return this->HealthzHandler(req, data, response);
+ };
+ webserver->RegisterUrlCallback("/healthz", healthz_callback);
Webserver::UrlCallback topics_callback =
bind<void>(mem_fn(&Statestore::TopicsHandler), this, _1, _2);
webserver->RegisterUrlCallback("/topics", "statestore_topics.tmpl",
@@ -1079,3 +1084,14 @@ void Statestore::ShutdownForTesting() {
int64_t Statestore::FailedExecutorDetectionTimeMs() {
return FLAGS_statestore_max_missed_heartbeats * FLAGS_statestore_heartbeat_frequency_ms;
}
+
+void Statestore::HealthzHandler(
+ const Webserver::WebRequest& req, std::stringstream* data, HttpStatusCode* response) {
+ if (service_started_) {
+ (*data) << "OK";
+ *response = HttpStatusCode::Ok;
+ return;
+ }
+ *(data) << "Not Available";
+ *response = HttpStatusCode::ServiceUnavailable;
+}
diff --git a/be/src/statestore/statestore.h b/be/src/statestore/statestore.h
index 7b6844d..e30840b 100644
--- a/be/src/statestore/statestore.h
+++ b/be/src/statestore/statestore.h
@@ -43,6 +43,8 @@
#include "util/thread-pool.h"
#include "util/webserver.h"
+using kudu::HttpStatusCode;
+
namespace impala {
class Status;
@@ -564,8 +566,8 @@ class Statestore : public CacheLineAligned {
/// Thread that monitors the heartbeats of all subscribers.
std::unique_ptr<Thread> heartbeat_monitoring_thread_;
- /// Flag to indicate that the statestore has been initialized.
- bool initialized_ = false;
+ /// Indicates whether the statestore has been initialized and the service is ready.
+ std::atomic_bool service_started_{false};
/// Cache of subscriber clients used for UpdateState() RPCs. Only one client per
/// subscriber should be used, but the cache helps with the client lifecycle on failure.
@@ -721,6 +723,10 @@ class Statestore : public CacheLineAligned {
/// last_heartbeat_ts_ has not been updated in that interval, it logs the subscriber's
/// id.
[[noreturn]] void MonitorSubscriberHeartbeat();
+
+ /// Raw callback to indicate whether the service is ready.
+ void HealthzHandler(const Webserver::WebRequest& req, std::stringstream* data,
+ HttpStatusCode* response);
};
} // namespace impala
diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py
index cab6108..7cc693c 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -50,6 +50,7 @@ class TestWebPage(ImpalaTestSuite):
BACKENDS_URL = "http://localhost:{0}/backends"
PROMETHEUS_METRICS_URL = "http://localhost:{0}/metrics_prometheus"
QUERIES_URL = "http://localhost:{0}/queries"
+ HEALTHZ_URL = "http://localhost:{0}/healthz"
# log4j changes do not apply to the statestore since it doesn't
# have an embedded JVM. So we make two sets of ports to test the
@@ -725,8 +726,9 @@ class TestWebPage(ImpalaTestSuite):
def test_healthz_endpoint(self):
"""Test to check that the /healthz endpoint returns 200 OK."""
- page = requests.get("http://localhost:25000/healthz")
- assert page.status_code == requests.codes.ok
+ for port in self.TEST_PORTS_WITH_SS:
+ page = requests.get(self.HEALTHZ_URL.format(port))
+ assert page.status_code == requests.codes.ok
def test_knox_compatibility(self):
"""Checks that the template files conform to the requirements for compatibility with
[impala] 03/03: IMPALA-10090 Pull newest code of native-toolchain
before build it
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6aaea3216ca2fabe352b9f11a7d140e9d9c183c9
Author: huangtianhua <hu...@gmail.com>
AuthorDate: Wed Sep 2 17:45:45 2020 +0800
IMPALA-10090 Pull newest code of native-toolchain before build it
If native-toolchain exists we should pull the newest code
before build it.
Change-Id: I2da3ffce7abb88190be0a5ea0e2cf603f98ee15e
Reviewed-on: http://gerrit.cloudera.org:8080/16402
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/bootstrap_system.sh | 3 +++
1 file changed, 3 insertions(+)
diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
index 2eca95f..7a51fb4 100755
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -486,6 +486,9 @@ if [[ $ARCH_NAME == 'aarch64' ]]; then
if ! [[ -d "$NATIVE_TOOLCHAIN_HOME" ]]; then
time -p git clone https://github.com/cloudera/native-toolchain/ \
"$NATIVE_TOOLCHAIN_HOME"
+ else
+ # pull newest code of native-toolchain if exists
+ git pull
fi
cd "$NATIVE_TOOLCHAIN_HOME"
echo "Begin build tool chain, may need several hours, please be patient...."
[impala] 01/03: IMPALA-10124 admission-controller-test fails with
no such file or directory error
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1e63722f8da7cd94a6937891832e51eae834c956
Author: Qifan Chen <qc...@cloudera.com>
AuthorDate: Wed Sep 2 15:06:14 2020 -0400
IMPALA-10124 admission-controller-test fails with no such file or
directory error
This work addresses a failure by disabling undefined behavior sanitizer
testing for AdmissionControllerTest.TopNQueryCheck test. In the test,
std::regex_match() is used to verify the appearance of certain strings
and can produce a core with very long stack trace failling in
std::vector::operator[]().
Testing:
1. Ran the test in both regular and disabling undefined behavior
sanitizer check modes. No core was seen.
Change-Id: I16d6cff8fad8d0e93a24ec3fefa9cc1f8c471aad
Reviewed-on: http://gerrit.cloudera.org:8080/16404
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/scheduling/admission-controller-test.cc | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/be/src/scheduling/admission-controller-test.cc b/be/src/scheduling/admission-controller-test.cc
index 16a22c3..3c05fed 100644
--- a/be/src/scheduling/admission-controller-test.cc
+++ b/be/src/scheduling/admission-controller-test.cc
@@ -956,7 +956,10 @@ TEST_F(AdmissionControllerTest, DedicatedCoordAdmissionChecks) {
/// Test that AdmissionController can identify 5 queries with top memory consumption
/// from 4 pools. Each pool holds a number of queries with different memory consumptions.
+/// Run the test only when undefined behavior sanitizer check is off to avoid core
+/// generated from std::regex_match().
TEST_F(AdmissionControllerTest, TopNQueryCheck) {
+#ifndef UNDEFINED_SANITIZER
// Pass the paths of the configuration files as command line flags
FLAGS_fair_scheduler_allocation_path = GetResourceFile("fair-scheduler-test2.xml");
FLAGS_llama_site_path = GetResourceFile("llama-site-test2.xml");
@@ -1109,6 +1112,7 @@ TEST_F(AdmissionControllerTest, TopNQueryCheck) {
// Reset the consumption_ counter for all trackers so that TearDown() call
// can run cleanly.
ResetMemConsumed(pool_mem_tracker->GetRootMemTracker());
+#endif
}
} // end namespace impala