You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2021/03/13 02:24:30 UTC
[impala] 06/08: IMPALA-10546: Add ImpalaServer interface to
retrieve BackendConfig from impalad
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 2e53c11484dd1f9044020b0035d9eab96d496a9c
Author: Kurt Deschler <kd...@cloudera.com>
AuthorDate: Mon Feb 24 23:46:34 2020 -0500
IMPALA-10546: Add ImpalaServer interface to retrieve BackendConfig from impalad
This patch add a new interface ImpalaServer::GetBackendConfig() that
returns the current TBackendGflags from impalad.
Testing:
Called new interface from external frontend. Verified that
TBackendGflags were populated correctly.
Reviewed-by: John Sherman <jf...@cloudera.com>
Change-Id: I14a3cee29f1fc91f4431b7ea89053bb3fbfa5e69
Reviewed-on: http://gerrit.cloudera.org:8080/17116
Reviewed-by: Thomas Tauber-Marshall <tm...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/catalog/catalog.cc | 2 +-
be/src/rpc/hs2-http-test.cc | 2 ++
be/src/service/frontend.cc | 2 +-
be/src/service/impala-hs2-server.cc | 22 ++++++++++++++++++++++
be/src/service/impala-server.h | 5 +++++
be/src/util/backend-gflag-util.cc | 9 +++++++--
be/src/util/backend-gflag-util.h | 5 ++++-
be/src/util/logging-support.cc | 2 +-
common/thrift/ImpalaService.thrift | 13 +++++++++++++
tests/hs2/test_hs2.py | 8 ++++++++
10 files changed, 64 insertions(+), 6 deletions(-)
diff --git a/be/src/catalog/catalog.cc b/be/src/catalog/catalog.cc
index d75484f..e918d4e 100644
--- a/be/src/catalog/catalog.cc
+++ b/be/src/catalog/catalog.cc
@@ -81,7 +81,7 @@ Catalog::Catalog() {
}
jbyteArray cfg_bytes;
- ABORT_IF_ERROR(GetThriftBackendGflags(jni_env, &cfg_bytes));
+ ABORT_IF_ERROR(GetThriftBackendGFlagsForJNI(jni_env, &cfg_bytes));
jobject catalog = jni_env->NewObject(catalog_class_, catalog_ctor_, cfg_bytes);
CLEAN_EXIT_IF_EXC(jni_env);
diff --git a/be/src/rpc/hs2-http-test.cc b/be/src/rpc/hs2-http-test.cc
index 2c95550..0c61df9 100644
--- a/be/src/rpc/hs2-http-test.cc
+++ b/be/src/rpc/hs2-http-test.cc
@@ -50,6 +50,8 @@ class TestHS2Service : public ImpalaHiveServer2ServiceIf {
TExecuteStatementResp& _return, const TExecuteStatementReq& req) {}
virtual void ExecutePlannedStatement(
TExecuteStatementResp& _return, const TExecutePlannedStatementReq& req) {}
+ virtual void GetBackendConfig(TGetBackendConfigResp& _return,
+ const TGetBackendConfigReq& req) {}
virtual void GetTypeInfo(TGetTypeInfoResp& _return, const TGetTypeInfoReq& req) {}
virtual void GetCatalogs(TGetCatalogsResp& _return, const TGetCatalogsReq& req) {}
virtual void GetSchemas(TGetSchemasResp& _return, const TGetSchemasReq& req) {}
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index af1339c..b5a3d1a 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -131,7 +131,7 @@ Frontend::Frontend() {
};
jbyteArray cfg_bytes;
- ABORT_IF_ERROR(GetThriftBackendGflags(jni_env, &cfg_bytes));
+ ABORT_IF_ERROR(GetThriftBackendGFlagsForJNI(jni_env, &cfg_bytes));
// Pass in whether this is a backend test, so that the Frontend can avoid certain
// unnecessary initialization that introduces dependencies on a running minicluster.
diff --git a/be/src/service/impala-hs2-server.cc b/be/src/service/impala-hs2-server.cc
index debd755..76a1c35 100644
--- a/be/src/service/impala-hs2-server.cc
+++ b/be/src/service/impala-hs2-server.cc
@@ -47,6 +47,7 @@
#include "service/query-options.h"
#include "service/query-result-set.h"
#include "util/auth-util.h"
+#include "util/backend-gflag-util.h"
#include "util/debug-util.h"
#include "util/impalad-metrics.h"
#include "util/metrics.h"
@@ -1175,6 +1176,27 @@ void ImpalaServer::GetDelegationToken(TGetDelegationTokenResp& return_val,
return_val.status.__set_errorMessage("Not implemented");
}
+void ImpalaServer::GetBackendConfig(TGetBackendConfigResp& return_val,
+ const TGetBackendConfigReq& req) {
+ VLOG_QUERY << "GetBackendConfig(): req=" << ThriftDebugString(req);
+ const ThriftServer::ConnectionContext* connection_context =
+ ThriftServer::GetThreadConnectionContext();
+ if (connection_context->server_name != EXTERNAL_FRONTEND_SERVER_NAME) {
+ HS2_RETURN_ERROR(return_val, "Unsupported operation",
+ SQLSTATE_OPTIONAL_FEATURE_NOT_IMPLEMENTED);
+ }
+ TUniqueId session_id;
+ TUniqueId secret;
+ HS2_RETURN_IF_ERROR(return_val, THandleIdentifierToTUniqueId(
+ req.sessionHandle.sessionId, &session_id, &secret), SQLSTATE_GENERAL_ERROR);
+ HS2_RETURN_IF_ERROR(return_val,
+ PopulateThriftBackendGflags(return_val.backend_config),
+ SQLSTATE_GENERAL_ERROR);
+
+ return_val.status.__set_statusCode(thrift::TStatusCode::SUCCESS_STATUS);
+ VLOG_RPC << "GetBackendConfig(): return_val=" << ThriftDebugString(return_val);
+}
+
void ImpalaServer::CancelDelegationToken(TCancelDelegationTokenResp& return_val,
const TCancelDelegationTokenReq& req) {
return_val.status.__set_statusCode(thrift::TStatusCode::ERROR_STATUS);
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index 9a3250e..cf63fa8 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -29,6 +29,7 @@
#include <boost/uuid/uuid_io.hpp>
#include "common/status.h"
+#include "gen-cpp/BackendGflags_types.h"
#include "gen-cpp/Frontend_types.h"
#include "gen-cpp/ImpalaHiveServer2Service.h"
#include "gen-cpp/ImpalaService.h"
@@ -352,6 +353,10 @@ class ImpalaServer : public ImpalaServiceIf,
apache::hive::service::cli::thrift::TExecuteStatementResp& return_val,
const TExecutePlannedStatementReq& req);
+ // Retrieves the current BackendConfig
+ virtual void GetBackendConfig(TGetBackendConfigResp& return_val,
+ const TGetBackendConfigReq& request);
+
/// Closes an Impala operation and returns additional information about the closed
/// operation.
virtual void CloseImpalaOperation(
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 2ef0311..d20c94e 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -163,8 +163,7 @@ Status GetConfigFromCommand(const string& flag_cmd, string& result) {
return Status::OK();
}
-Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
- TBackendGflags cfg;
+Status PopulateThriftBackendGflags(TBackendGflags& cfg) {
cfg.__set_load_catalog_in_background(FLAGS_load_catalog_in_background);
cfg.__set_enable_orc_scanner(FLAGS_enable_orc_scanner);
cfg.__set_use_local_catalog(FLAGS_use_local_catalog);
@@ -269,6 +268,12 @@ Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
cfg.__set_saml2_group_attribute_name(FLAGS_saml2_group_attribute_name);
cfg.__set_saml2_group_filter(FLAGS_saml2_group_filter);
cfg.__set_saml2_ee_test_mode(FLAGS_saml2_ee_test_mode);
+ return Status::OK();
+}
+
+Status GetThriftBackendGFlagsForJNI(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
+ TBackendGflags cfg;
+ RETURN_IF_ERROR(PopulateThriftBackendGflags(cfg));
RETURN_IF_ERROR(SerializeThriftMsg(jni_env, &cfg, cfg_bytes));
return Status::OK();
}
diff --git a/be/src/util/backend-gflag-util.h b/be/src/util/backend-gflag-util.h
index d68ed2f..da17fd6 100644
--- a/be/src/util/backend-gflag-util.h
+++ b/be/src/util/backend-gflag-util.h
@@ -24,9 +24,12 @@
namespace impala {
+class TBackendGflags;
+
+Status PopulateThriftBackendGflags(TBackendGflags& cfg);
/// Builds the TBackendGflags object to pass to JNI. This is used to pass the gflag
/// configs to the Frontend and the Catalog.
-Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes);
+Status GetThriftBackendGFlagsForJNI(JNIEnv* jni_env, jbyteArray* cfg_bytes);
}
#endif
diff --git a/be/src/util/logging-support.cc b/be/src/util/logging-support.cc
index 2b6f9dd..459f913 100644
--- a/be/src/util/logging-support.cc
+++ b/be/src/util/logging-support.cc
@@ -81,7 +81,7 @@ Java_org_apache_impala_util_NativeLogger_Log(
namespace {
// Defaults to startup flag --v. FLAGS_v can be overriden at runtime for
// debugging, so we save the original value here in case we need to restore
-// the defaults. Set in GetThriftBackendGflags().
+// the defaults. Set in GetThriftBackendGFlagsForJNI().
int FLAGS_v_original_value;
static jclass log4j_logger_class_;
diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift
index c403055..44cc8a2 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -25,6 +25,7 @@ include "beeswax.thrift"
include "TCLIService.thrift"
include "RuntimeProfile.thrift"
include "Frontend.thrift"
+include "BackendGflags.thrift"
// ImpalaService accepts query execution options through beeswax.Query.configuration in
// key:value form. For example, the list of strings could be:
@@ -823,6 +824,15 @@ struct TExecutePlannedStatementReq {
2: required Frontend.TExecRequest plan
}
+struct TGetBackendConfigReq {
+ 1: required TCLIService.TSessionHandle sessionHandle
+}
+
+struct TGetBackendConfigResp {
+ 1: required TCLIService.TStatus status
+
+ 2: required BackendGflags.TBackendGflags backend_config
+}
service ImpalaHiveServer2Service extends TCLIService.TCLIService {
// Returns the exec summary for the given query. The exec summary is only valid for
@@ -843,4 +853,7 @@ service ImpalaHiveServer2Service extends TCLIService.TCLIService {
// Execute statement with supplied ExecRequest
TCLIService.TExecuteStatementResp ExecutePlannedStatement(
1:TExecutePlannedStatementReq req);
+
+ // Returns the current TBackendGflags. Only supported for the "external fe" service.
+ TGetBackendConfigResp GetBackendConfig(1:TGetBackendConfigReq req);
}
diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py
index 38c100e..b72e221 100644
--- a/tests/hs2/test_hs2.py
+++ b/tests/hs2/test_hs2.py
@@ -732,6 +732,14 @@ class TestHS2(HS2TestSuite):
assert len(exec_summary_resp.summary.nodes) > 0
@needs_session()
+ def test_get_backend_config(self):
+ get_backend_config_req = ImpalaHiveServer2Service.TGetBackendConfigReq()
+ get_backend_config_req.sessionHandle = self.session_handle
+ get_backend_config_resp = self.hs2_client.GetBackendConfig(get_backend_config_req)
+ TestHS2.check_response(get_backend_config_resp,
+ TCLIService.TStatusCode.ERROR_STATUS, "Unsupported operation")
+
+ @needs_session()
def test_get_profile(self):
statement = "SELECT COUNT(2) FROM functional.alltypes"
execute_statement_resp = self.execute_statement(statement)