You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/11/17 16:09:47 UTC
[36/50] [abbrv] incubator-impala git commit: IMPALA-3552: Make
incremental stats max serialized size configurable
IMPALA-3552: Make incremental stats max serialized size configurable
The fix "IMPALA-2648/IMPALA-2664" introduced a conservative limitation
on the maximum serialized size of incremental stats. As a side effect,
some users with very large tables are experiencing regressions
especially when they upgrade impala and the serialized size goes
beyond 200MB.
To mitigate the issue, the change introduces a new gflag,
'inc_stats_size_limit_bytes' to make the max serialized size
configurable, which allows impala users to specify their own maximum
serialized size. Default value for inc_stats_size_limit_bytes is
200MB.
The change introduces a TBackendGflags class to pass the gflags from
backend to the Frontend and the Catalog via thrift. This also revamps
existing query options to use the TBackendConfig.
Change-Id: I33684725a61eabc67237503e61178305d37d3cb5
Reviewed-on: http://gerrit.cloudera.org:8080/4867
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/3f2f008a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/3f2f008a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/3f2f008a
Branch: refs/heads/hadoop-next
Commit: 3f2f008ac4365d7f7c616730d3faeebd776b74d2
Parents: cac02d6
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Fri Nov 4 16:36:39 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 03:22:11 2016 +0000
----------------------------------------------------------------------
be/generated-sources/gen-cpp/CMakeLists.txt | 2 +
be/src/catalog/catalog.cc | 28 ++------
be/src/common/global-flags.cc | 6 ++
be/src/service/fe-support.cc | 18 -----
be/src/service/frontend.cc | 38 +++-------
be/src/util/CMakeLists.txt | 1 +
be/src/util/backend-gflag-util.cc | 76 ++++++++++++++++++++
be/src/util/backend-gflag-util.h | 31 ++++++++
common/thrift/BackendGflags.thrift | 55 ++++++++++++++
common/thrift/CMakeLists.txt | 1 +
common/thrift/Frontend.thrift | 5 --
.../apache/impala/analysis/AlterViewStmt.java | 3 +-
.../impala/analysis/ComputeStatsStmt.java | 10 +--
.../apache/impala/analysis/CreateViewStmt.java | 3 +-
.../org/apache/impala/authorization/User.java | 26 ++-----
.../org/apache/impala/catalog/HdfsTable.java | 11 +--
.../org/apache/impala/catalog/KuduTable.java | 2 +-
.../org/apache/impala/common/RuntimeEnv.java | 12 ----
.../java/org/apache/impala/planner/Planner.java | 3 +-
.../apache/impala/service/BackendConfig.java | 70 ++++++++++--------
.../org/apache/impala/service/FeSupport.java | 18 -----
.../org/apache/impala/service/JniCatalog.java | 30 ++++----
.../org/apache/impala/service/JniFrontend.java | 26 +++----
.../java/org/apache/impala/util/KuduUtil.java | 4 +-
24 files changed, 280 insertions(+), 199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/generated-sources/gen-cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/generated-sources/gen-cpp/CMakeLists.txt b/be/generated-sources/gen-cpp/CMakeLists.txt
index 35f7d65..ea7735f 100644
--- a/be/generated-sources/gen-cpp/CMakeLists.txt
+++ b/be/generated-sources/gen-cpp/CMakeLists.txt
@@ -39,6 +39,8 @@ set(SRC_FILES
ImpalaHiveServer2Service.cpp
beeswax_constants.cpp
beeswax_types.cpp
+ BackendGflags_constants.cpp
+ BackendGflags_types.cpp
BeeswaxService.cpp
TCLIService.cpp
TCLIService_constants.cpp
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/catalog/catalog.cc
----------------------------------------------------------------------
diff --git a/be/src/catalog/catalog.cc b/be/src/catalog/catalog.cc
index 7d6fdca..b3690f6 100644
--- a/be/src/catalog/catalog.cc
+++ b/be/src/catalog/catalog.cc
@@ -22,15 +22,12 @@
#include "common/logging.h"
#include "rpc/jni-thrift-util.h"
-#include "util/logging-support.h"
+#include "util/backend-gflag-util.h"
#include "common/names.h"
using namespace impala;
-DECLARE_bool(load_auth_to_local_rules);
-DECLARE_string(principal);
-DECLARE_string(local_library_dir);
DEFINE_bool(load_catalog_in_background, false,
"If true, loads catalog metadata in the background. If false, metadata is loaded "
@@ -41,13 +38,9 @@ DEFINE_int32(num_metadata_loading_threads, 16,
DEFINE_string(sentry_config, "", "Local path to a sentry-site.xml configuration "
"file. If set, authorization will be enabled.");
-DECLARE_int32(non_impala_java_vlog);
-DECLARE_int32(kudu_operation_timeout_ms);
-
Catalog::Catalog() {
JniMethodDescriptor methods[] = {
- {"<init>", "(ZILjava/lang/String;IIZLjava/lang/String;Ljava/lang/String;I)V",
- &catalog_ctor_},
+ {"<init>", "([B)V", &catalog_ctor_},
{"updateCatalog", "([B)[B", &update_metastore_id_},
{"execDdl", "([B)[B", &exec_ddl_id_},
{"resetMetadata", "([B)[B", &reset_metadata_id_},
@@ -70,19 +63,10 @@ Catalog::Catalog() {
ABORT_IF_ERROR(JniUtil::LoadJniMethod(jni_env, catalog_class_, &(methods[i])));
}
- jboolean load_in_background = FLAGS_load_catalog_in_background;
- jint num_metadata_loading_threads = FLAGS_num_metadata_loading_threads;
- jstring sentry_config = jni_env->NewStringUTF(FLAGS_sentry_config.c_str());
- // auth_to_local rules are read if --load_auth_to_local_rules is set to true
- // and impala is kerberized.
- jboolean auth_to_local = FLAGS_load_auth_to_local_rules && !FLAGS_principal.empty();
- jstring principal = jni_env->NewStringUTF(FLAGS_principal.c_str());
- jstring local_library_dir = jni_env->NewStringUTF(FLAGS_local_library_dir.c_str());
- jint kudu_operation_timeout = FLAGS_kudu_operation_timeout_ms;
- jobject catalog = jni_env->NewObject(catalog_class_, catalog_ctor_,
- load_in_background, num_metadata_loading_threads, sentry_config,
- FlagToTLogLevel(FLAGS_v), FlagToTLogLevel(FLAGS_non_impala_java_vlog),
- auth_to_local, principal, local_library_dir, kudu_operation_timeout);
+ jbyteArray cfg_bytes;
+ ABORT_IF_ERROR(GetThriftBackendGflags(jni_env, &cfg_bytes));
+
+ jobject catalog = jni_env->NewObject(catalog_class_, catalog_ctor_, cfg_bytes);
EXIT_IF_EXC(jni_env);
ABORT_IF_ERROR(JniUtil::LocalToGlobalRef(jni_env, catalog, &catalog_));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/common/global-flags.cc
----------------------------------------------------------------------
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index f75236c..5a3eff4 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -123,3 +123,9 @@ DEFINE_bool(enable_accept_queue_server, true,
"TThreadedServer that accepts connections as quickly as possible and hands them off "
"to a thread pool to finish setup, reducing the chances that connections time out "
"waiting to be accepted.");
+
+DEFINE_int64(inc_stats_size_limit_bytes, 200 * (1LL<<20), "Maximum size of "
+ "incremental stats the catalog is allowed to serialize per table. "
+ "This limit is set as a safety check, to prevent the JVM from "
+ "hitting a maximum array limit of 1GB (or OOM) while building "
+ "the thrift objects to send to impalads. By default, it's set to 200MB");
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/service/fe-support.cc
----------------------------------------------------------------------
diff --git a/be/src/service/fe-support.cc b/be/src/service/fe-support.cc
index 42ceea2..7147964 100644
--- a/be/src/service/fe-support.cc
+++ b/be/src/service/fe-support.cc
@@ -336,20 +336,6 @@ Java_org_apache_impala_service_FeSupport_NativePrioritizeLoad(
return result_bytes;
}
-extern "C"
-JNIEXPORT jbyteArray JNICALL
-Java_org_apache_impala_service_FeSupport_NativeGetStartupOptions(JNIEnv* env,
- jclass caller_class) {
- TStartupOptions options;
- ExecEnv* exec_env = ExecEnv::GetInstance();
- ImpalaServer* impala_server = exec_env->impala_server();
- options.__set_compute_lineage(impala_server->IsLineageLoggingEnabled());
- jbyteArray result_bytes = NULL;
- THROW_IF_ERROR_RET(SerializeThriftMsg(env, &options, &result_bytes), env,
- JniUtil::internal_exc_class(), result_bytes);
- return result_bytes;
-}
-
namespace impala {
static JNINativeMethod native_methods[] = {
@@ -373,10 +359,6 @@ static JNINativeMethod native_methods[] = {
(char*)"NativePrioritizeLoad", (char*)"([B)[B",
(void*)::Java_org_apache_impala_service_FeSupport_NativePrioritizeLoad
},
- {
- (char*)"NativeGetStartupOptions", (char*)"()[B",
- (void*)::Java_org_apache_impala_service_FeSupport_NativeGetStartupOptions
- },
};
void InitFeSupport() {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/service/frontend.cc
----------------------------------------------------------------------
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index f8cecfb..b83d091 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -22,21 +22,13 @@
#include "common/logging.h"
#include "rpc/jni-thrift-util.h"
+#include "util/backend-gflag-util.h"
#include "util/jni-util.h"
-#include "util/logging-support.h"
#include "common/names.h"
using namespace impala;
-DECLARE_string(sentry_config);
-DECLARE_int32(non_impala_java_vlog);
-DECLARE_bool(load_auth_to_local_rules);
-DECLARE_string(principal);
-DECLARE_int32(kudu_operation_timeout_ms);
-
-DEFINE_bool(load_catalog_at_startup, false, "if true, load all catalog data at startup");
-
// Authorization related flags. Must be set to valid values to properly configure
// authorization.
DEFINE_string(server_name, "", "The name to use for securing this impalad "
@@ -62,10 +54,10 @@ DEFINE_string(authorized_proxy_user_config_delimiter, ",",
DEFINE_string(kudu_master_hosts, "", "Specifies the default Kudu master(s). The given "
"value should be a comma separated list of hostnames or IP addresses; ports are "
"optional.");
+
Frontend::Frontend() {
JniMethodDescriptor methods[] = {
- {"<init>", "(ZLjava/lang/String;Ljava/lang/String;Ljava/lang/String;"
- "Ljava/lang/String;IIZLjava/lang/String;I)V", &fe_ctor_},
+ {"<init>", "([B)V", &fe_ctor_},
{"createExecRequest", "([B)[B", &create_exec_request_id_},
{"getExplainPlan", "([B)Ljava/lang/String;", &get_explain_plan_id_},
{"getHadoopConfig", "([B)[B", &get_hadoop_config_id_},
@@ -89,7 +81,7 @@ Frontend::Frontend() {
{"loadTableData", "([B)[B", &load_table_data_id_},
{"getTableFiles", "([B)[B", &get_table_files_id_},
{"showCreateFunction", "([B)Ljava/lang/String;", &show_create_function_id_},
- {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_},
+ {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_}
};
JNIEnv* jni_env = getJNIEnv();
@@ -102,24 +94,10 @@ Frontend::Frontend() {
ABORT_IF_ERROR(JniUtil::LoadJniMethod(jni_env, fe_class_, &(methods[i])));
};
- jboolean lazy = (FLAGS_load_catalog_at_startup ? false : true);
- jstring policy_file_path =
- jni_env->NewStringUTF(FLAGS_authorization_policy_file.c_str());
- jstring server_name =
- jni_env->NewStringUTF(FLAGS_server_name.c_str());
- jstring sentry_config =
- jni_env->NewStringUTF(FLAGS_sentry_config.c_str());
- jstring auth_provider_class =
- jni_env->NewStringUTF(FLAGS_authorization_policy_provider_class.c_str());
- // auth_to_local rules are read if --load_auth_to_local_rules is set to true
- // and impala is kerberized.
- jboolean auth_to_local = FLAGS_load_auth_to_local_rules && !FLAGS_principal.empty();
- jstring kudu_master_hosts = jni_env->NewStringUTF(FLAGS_kudu_master_hosts.c_str());
- jint kudu_operation_timeout = FLAGS_kudu_operation_timeout_ms;
- jobject fe = jni_env->NewObject(fe_class_, fe_ctor_, lazy, server_name,
- policy_file_path, sentry_config, auth_provider_class, FlagToTLogLevel(FLAGS_v),
- FlagToTLogLevel(FLAGS_non_impala_java_vlog), auth_to_local, kudu_master_hosts,
- kudu_operation_timeout);
+ jbyteArray cfg_bytes;
+ ABORT_IF_ERROR(GetThriftBackendGflags(jni_env, &cfg_bytes));
+
+ jobject fe = jni_env->NewObject(fe_class_, fe_ctor_, cfg_bytes);
EXIT_IF_EXC(jni_env);
ABORT_IF_ERROR(JniUtil::LocalToGlobalRef(jni_env, fe, &fe_));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index ecc222a..6656a38 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -32,6 +32,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/util")
add_library(Util
auth-util.cc
avro-util.cc
+ backend-gflag-util.cc
benchmark.cc
bitmap.cc
bit-util.cc
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/backend-gflag-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
new file mode 100644
index 0000000..f504e57
--- /dev/null
+++ b/be/src/util/backend-gflag-util.cc
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/backend-gflag-util.h"
+
+#include "gen-cpp/BackendGflags_types.h"
+#include "rpc/jni-thrift-util.h"
+#include "util/logging-support.h"
+
+#include <gflags/gflags.h>
+
+// Configs for the Frontend and the Catalog.
+DECLARE_bool(load_catalog_in_background);
+DECLARE_bool(load_auth_to_local_rules);
+DECLARE_int32(non_impala_java_vlog);
+DECLARE_int32(read_size);
+DECLARE_int32(num_metadata_loading_threads);
+DECLARE_int32(kudu_operation_timeout_ms);
+DECLARE_int64(inc_stats_size_limit_bytes);
+DECLARE_string(principal);
+DECLARE_string(lineage_event_log_dir);
+DECLARE_string(principal);
+DECLARE_string(local_library_dir);
+DECLARE_string(server_name);
+DECLARE_string(authorization_policy_file);
+DECLARE_string(authorization_policy_provider_class);
+DECLARE_string(authorized_proxy_user_config);
+DECLARE_string(authorized_proxy_user_config_delimiter);
+DECLARE_string(kudu_master_hosts);
+DECLARE_string(sentry_config);
+DECLARE_string(sentry_config);
+
+namespace impala {
+
+Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
+ TBackendGflags cfg;
+ cfg.__set_authorization_policy_file(FLAGS_authorization_policy_file);
+ cfg.__set_load_catalog_in_background(FLAGS_load_catalog_in_background);
+ cfg.__set_server_name(FLAGS_server_name);
+ cfg.__set_sentry_config(FLAGS_sentry_config);
+ cfg.__set_authorization_policy_provider_class(
+ FLAGS_authorization_policy_provider_class);
+ cfg.__set_kudu_master_hosts(FLAGS_kudu_master_hosts);
+ cfg.__set_read_size(FLAGS_read_size);
+ cfg.__set_num_metadata_loading_threads(FLAGS_num_metadata_loading_threads);
+ cfg.__set_sentry_config(FLAGS_sentry_config);
+ // auth_to_local rules are read if --load_auth_to_local_rules is set to true
+ // and impala is kerberized.
+ cfg.__set_load_auth_to_local_rules(FLAGS_load_auth_to_local_rules);
+ cfg.__set_principal(FLAGS_principal);
+ cfg.__set_impala_log_lvl(FlagToTLogLevel(FLAGS_v));
+ cfg.__set_non_impala_java_vlog(FlagToTLogLevel(FLAGS_non_impala_java_vlog));
+ cfg.__set_inc_stats_size_limit_bytes(FLAGS_inc_stats_size_limit_bytes);
+ cfg.__set_lineage_event_log_dir(FLAGS_lineage_event_log_dir);
+ cfg.__set_local_library_path(FLAGS_local_library_dir);
+ cfg.__set_kudu_operation_timeout_ms(FLAGS_kudu_operation_timeout_ms);
+ JniLocalFrame jni_frame;
+ RETURN_IF_ERROR(jni_frame.push(jni_env));
+ RETURN_IF_ERROR(SerializeThriftMsg(jni_env, &cfg, cfg_bytes));
+ return Status::OK();
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/backend-gflag-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/backend-gflag-util.h b/be/src/util/backend-gflag-util.h
new file mode 100644
index 0000000..796a2d5
--- /dev/null
+++ b/be/src/util/backend-gflag-util.h
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef UTIL_BACKEND_CONFIG_H_
+#define UTIL_BACKEND_CONFIG_H_
+
+#include "common/status.h"
+#include "util/jni-util.h"
+
+namespace impala {
+
+/// Builds the TBackendGflags object to pass to JNI. This is used to pass the gflag
+/// configs to the Frontend and the Catalog.
+Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes);
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/BackendGflags.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
new file mode 100644
index 0000000..ad081a6
--- /dev/null
+++ b/common/thrift/BackendGflags.thrift
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace cpp impala
+namespace java org.apache.impala.thrift
+
+// Used to pass gflags from backend to frontend, JniCatalog and JniFrontend
+// Attributes without comments correspond to gflags
+struct TBackendGflags {
+ 1: required string sentry_config
+
+ 2: required bool load_auth_to_local_rules
+
+ 3: required i32 non_impala_java_vlog
+
+ 4: required i32 impala_log_lvl
+
+ 5: required i64 inc_stats_size_limit_bytes
+
+ 6: required string lineage_event_log_dir
+
+ 7: required bool load_catalog_in_background
+
+ 8: required i32 num_metadata_loading_threads
+
+ 9: required string principal
+
+ 10: required string authorization_policy_file
+
+ 11: required string server_name
+
+ 12: required string authorization_policy_provider_class
+
+ 13: required string kudu_master_hosts
+
+ 14: required string local_library_path
+
+ 15: required i32 read_size
+
+ 16: required i32 kudu_operation_timeout_ms
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt
index 10d363d..1512e52 100644
--- a/common/thrift/CMakeLists.txt
+++ b/common/thrift/CMakeLists.txt
@@ -144,6 +144,7 @@ set (EXT_DATA_SRC_FILES
set (SRC_FILES
ErrorCodes.thrift
beeswax.thrift
+ BackendGflags.thrift
CatalogInternalService.thrift
CatalogObjects.thrift
CatalogService.thrift
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/Frontend.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index f6ba35a..8c279f5 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -721,11 +721,6 @@ struct TGetAllHadoopConfigsResponse {
1: optional map<string, string> configs;
}
-// BE startup options
-struct TStartupOptions {
- 1: optional bool compute_lineage
-}
-
// For creating a test descriptor table. The tuples and their memory layout are computed
// in the FE.
struct TBuildTestDescriptorTableParams {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
index e7d3f66..6740f14 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
@@ -22,6 +22,7 @@ import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.View;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.RuntimeEnv;
+import org.apache.impala.service.BackendConfig;
import com.google.common.base.Preconditions;
@@ -52,7 +53,7 @@ public class AlterViewStmt extends CreateOrAlterViewStmtBase {
}
createColumnAndViewDefs(analyzer);
- if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+ if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
computeLineageGraph(analyzer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index 467faca..5ad3cad 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -33,6 +33,7 @@ import org.apache.impala.catalog.Type;
import org.apache.impala.catalog.View;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.PrintUtils;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TComputeStatsParams;
import org.apache.impala.thrift.TPartitionStats;
import org.apache.impala.thrift.TTableName;
@@ -284,16 +285,17 @@ public class ComputeStatsStmt extends StatementBase {
}
}
// For incremental stats, estimate the size of intermediate stats and report an
- // error if the estimate is greater than MAX_INCREMENTAL_STATS_SIZE_BYTES.
+ // error if the estimate is greater than --inc_stats_size_limit_bytes in bytes
if (isIncremental_) {
+ long incStatMaxSize = BackendConfig.INSTANCE.getIncStatsMaxSize();
long statsSizeEstimate = hdfsTable.getColumns().size() *
hdfsTable.getPartitions().size() * HdfsTable.STATS_SIZE_PER_COLUMN_BYTES;
- if (statsSizeEstimate > HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES) {
+ if (statsSizeEstimate > incStatMaxSize) {
LOG.error("Incremental stats size estimate for table " + hdfsTable.getName() +
- " exceeded " + HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES + ", estimate = "
+ " exceeded " + incStatMaxSize + ", estimate = "
+ statsSizeEstimate);
throw new AnalysisException("Incremental stats size estimate exceeds "
- + PrintUtils.printBytes(HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES)
+ + PrintUtils.printBytes(incStatMaxSize)
+ ". Please try COMPUTE STATS instead.");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
index 6e30aeb..6b90083 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
@@ -24,6 +24,7 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.thrift.TAccessEvent;
import org.apache.impala.thrift.TCatalogObjectType;
+import org.apache.impala.service.BackendConfig;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@@ -60,7 +61,7 @@ public class CreateViewStmt extends CreateOrAlterViewStmtBase {
TCatalogObjectType.VIEW, Privilege.CREATE.toString()));
createColumnAndViewDefs(analyzer);
- if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+ if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
computeLineageGraph(analyzer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/authorization/User.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/User.java b/fe/src/main/java/org/apache/impala/authorization/User.java
index 190419b..e29b19f 100644
--- a/fe/src/main/java/org/apache/impala/authorization/User.java
+++ b/fe/src/main/java/org/apache/impala/authorization/User.java
@@ -19,42 +19,24 @@ package org.apache.impala.authorization;
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.service.BackendConfig;
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import org.apache.hadoop.security.authentication.util.KerberosName;
/*
* Class that represents a User of an Impala session.
*/
public class User {
- static {
- // If auth_to_local is enabled, we read the configuration hadoop.security.auth_to_local
- // from core-site.xml and use it for principal to short name conversion. If it is not,
- // we use the defaultRule ("RULE:[1:$1] RULE:[2:$1]"), which just extracts the user
- // name from any principal of form a@REALM or a/b@REALM. If auth_to_local is enabled
- // and hadoop.security.auth_to_local is not specified in the hadoop configs, we use
- // the "DEFAULT" rule that just extracts the username from any principal in the
- // cluster's local realm. For more details on principal to short name translation,
- // refer to org.apache.hadoop.security.KerberosName.
- final String defaultRule = "RULE:[1:$1] RULE:[2:$1]";
- final Configuration conf = new Configuration();
- if (BackendConfig.isAuthToLocalEnabled()) {
- KerberosName.setRules(conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL, "DEFAULT"));
- } else {
- // just extract the simple user name
- KerberosName.setRules(defaultRule);
- }
- }
-
private final String name_;
+ // Refer to BackendConfig.initAuthToLocal() for initialization
+ // of static auth_to_local configs in KerberosName class.
private KerberosName kerberosName_;
public User(String name) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 97831d7..69b4b4e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -63,6 +63,7 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.Pair;
import org.apache.impala.common.PrintUtils;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.ImpalaInternalServiceConstants;
import org.apache.impala.thrift.TAccessLevel;
import org.apache.impala.thrift.TCatalogObjectType;
@@ -169,12 +170,6 @@ public class HdfsTable extends Table {
// Store all the partition ids of an HdfsTable.
private HashSet<Long> partitionIds_ = Sets.newHashSet();
- // Maximum size (in bytes) of incremental stats the catalog is allowed to serialize per
- // table. This limit is set as a safety check, to prevent the JVM from hitting a
- // maximum array limit of 1GB (or OOM) while building the thrift objects to send to
- // impalads.
- public static final long MAX_INCREMENTAL_STATS_SIZE_BYTES = 200 * 1024 * 1024;
-
// Estimate (in bytes) of the incremental stats size per column per partition
public static final long STATS_SIZE_PER_COLUMN_BYTES = 400;
@@ -1593,7 +1588,7 @@ public class HdfsTable extends Table {
* partitions). To prevent the catalog from hitting an OOM error while trying to
* serialize large partition incremental stats, we estimate the stats size and filter
* the incremental stats data from partition objects if the estimate exceeds
- * MAX_INCREMENTAL_STATS_SIZE_BYTES.
+ * --inc_stats_size_limit_bytes
*/
private THdfsTable getTHdfsTable(boolean includeFileDesc, Set<Long> refPartitions) {
// includeFileDesc implies all partitions should be included (refPartitions == null).
@@ -1603,7 +1598,7 @@ public class HdfsTable extends Table {
long statsSizeEstimate =
numPartitions * getColumns().size() * STATS_SIZE_PER_COLUMN_BYTES;
boolean includeIncrementalStats =
- (statsSizeEstimate < MAX_INCREMENTAL_STATS_SIZE_BYTES);
+ (statsSizeEstimate < BackendConfig.INSTANCE.getIncStatsMaxSize());
Map<Long, THdfsPartition> idToPartition = Maps.newHashMap();
for (HdfsPartition partition: partitionMap_.values()) {
long id = partition.getId();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
index 6c30077..7b906a7 100644
--- a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
@@ -340,7 +340,7 @@ public class KuduTable extends Table {
try (KuduClient client = KuduUtil.createKuduClient(getKuduMasterHosts())) {
org.apache.kudu.client.KuduTable kuduTable = client.openTable(kuduTableName_);
List<LocatedTablet> tablets =
- kuduTable.getTabletsLocations(BackendConfig.getKuduClientTimeoutMs());
+ kuduTable.getTabletsLocations(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
for (LocatedTablet tab: tablets) {
TResultRowBuilder builder = new TResultRowBuilder();
builder.add("-1"); // The Kudu client API doesn't expose tablet row counts.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
index af58b16..d33455a 100644
--- a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
+++ b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
@@ -21,7 +21,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.impala.service.FeSupport;
-import org.apache.impala.thrift.TStartupOptions;
/**
* Contains runtime-specific parameters such as the number of CPU cores. Currently only
@@ -35,21 +34,11 @@ public class RuntimeEnv {
private int numCores_;
- // Indicates if column lineage information should be computed for each query.
- private boolean computeLineage_;
-
// Indicates whether this is an environment for testing.
private boolean isTestEnv_;
public RuntimeEnv() {
reset();
- try {
- TStartupOptions opts = FeSupport.GetStartupOptions();
- computeLineage_ = opts.compute_lineage;
- } catch (InternalException e) {
- LOG.error("Error retrieving BE startup options. Shutting down JVM");
- System.exit(1);
- }
}
/**
@@ -63,7 +52,6 @@ public class RuntimeEnv {
public void setNumCores(int numCores) { this.numCores_ = numCores; }
public void setTestEnv(boolean v) { isTestEnv_ = v; }
public boolean isTestEnv() { return isTestEnv_; }
- public boolean computeLineage() { return computeLineage_; }
public boolean isKuduSupported() {
return "true".equals(System.getenv("KUDU_IS_SUPPORTED"));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/planner/Planner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/Planner.java b/fe/src/main/java/org/apache/impala/planner/Planner.java
index ec4cf7e..52524fb 100644
--- a/fe/src/main/java/org/apache/impala/planner/Planner.java
+++ b/fe/src/main/java/org/apache/impala/planner/Planner.java
@@ -39,6 +39,7 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.common.TreeNode;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TExplainLevel;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TQueryExecRequest;
@@ -179,7 +180,7 @@ public class Planner {
ctx_.getRootAnalyzer().getTimeline().markEvent("Distributed plan created");
ColumnLineageGraph graph = ctx_.getRootAnalyzer().getColumnLineageGraph();
- if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+ if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
// Compute the column lineage graph
if (ctx_.isInsertOrCtas()) {
Table targetTable = ctx_.getAnalysisResult().getInsertStmt().getTargetTable();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index c90f674..47c987e 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -18,46 +18,60 @@
package org.apache.impala.service;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+
+import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.impala.thrift.TBackendGflags;
/**
* This class is meant to provide the FE with impalad backend configuration parameters,
* including command line arguments.
- * TODO: Remove this class and either
- * a) Figure out if there's a standard way to access flags from java
- * b) Create a util/gflags.java that let's us access the be flags
*/
public class BackendConfig {
- public static BackendConfig INSTANCE = new BackendConfig();
-
- // Default read block size (in bytes). This is the same as
- // the default FLAGS_read_size used by the IO manager in the backend.
- private final long READ_SIZE;
-
- // Variables below are overriden by JniFrontend/JniCatalog with user set configuration.
- // TODO: Read from backend instead of using static variables.
-
- // Determines how principal to short name conversion works. See User.java for more info.
- private static boolean allowAuthToLocalRules_ = false;
+ public static BackendConfig INSTANCE;
- // Kudu client timeout (ms).
- private static int kuduOperationTimeoutMs_ = 3 * 60 * 1000;
+ private TBackendGflags backendCfg_;
- private BackendConfig() {
- // TODO: Populate these by making calls to the backend instead of default constants.
- READ_SIZE = 8 * 1024 * 1024L;
+ private BackendConfig(TBackendGflags cfg) {
+ backendCfg_ = cfg;
}
- public long getReadSize() { return READ_SIZE; }
-
- public static boolean isAuthToLocalEnabled() { return allowAuthToLocalRules_; }
- public static void setAuthToLocal(boolean authToLocal) {
- allowAuthToLocalRules_ = authToLocal;
+ public static void create(TBackendGflags cfg) {
+ Preconditions.checkNotNull(cfg);
+ INSTANCE = new BackendConfig(cfg);
+ initAuthToLocal();
}
- public static int getKuduClientTimeoutMs() { return kuduOperationTimeoutMs_; }
+ public long getReadSize() { return backendCfg_.read_size; }
+ public boolean getComputeLineage() {
+ return !Strings.isNullOrEmpty(backendCfg_.lineage_event_log_dir);
+ }
+ public long getIncStatsMaxSize() { return backendCfg_.inc_stats_size_limit_bytes; }
+ public boolean isAuthToLocalEnabled() {
+ return backendCfg_.load_auth_to_local_rules &&
+ !Strings.isNullOrEmpty(backendCfg_.principal);
+ }
+ public int getKuduClientTimeoutMs() { return backendCfg_.kudu_operation_timeout_ms; }
- public static void setKuduClientTimeoutMs(int kuduOperationTimeoutMs) {
- Preconditions.checkArgument(kuduOperationTimeoutMs > 0);
- BackendConfig.kuduOperationTimeoutMs_ = kuduOperationTimeoutMs;
+ // Inits the auth_to_local configuration in the static KerberosName class.
+ private static void initAuthToLocal() {
+ // If auth_to_local is enabled, we read the configuration hadoop.security.auth_to_local
+ // from core-site.xml and use it for principal to short name conversion. If it is not,
+ // we use the defaultRule ("RULE:[1:$1] RULE:[2:$1]"), which just extracts the user
+ // name from any principal of form a@REALM or a/b@REALM. If auth_to_local is enabled
+ // and hadoop.security.auth_to_local is not specified in the hadoop configs, we use
+ // the "DEFAULT" rule that just extracts the username from any principal in the
+ // cluster's local realm. For more details on principal to short name translation,
+ // refer to org.apache.hadoop.security.KerberosName.
+ final String defaultRule = "RULE:[1:$1] RULE:[2:$1]";
+ final Configuration conf = new Configuration();
+ if (INSTANCE.isAuthToLocalEnabled()) {
+ KerberosName.setRules(conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL, "DEFAULT"));
+ } else {
+ // just extract the simple user name
+ KerberosName.setRules(defaultRule);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/FeSupport.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/FeSupport.java b/fe/src/main/java/org/apache/impala/service/FeSupport.java
index f982df3..c578726 100644
--- a/fe/src/main/java/org/apache/impala/service/FeSupport.java
+++ b/fe/src/main/java/org/apache/impala/service/FeSupport.java
@@ -45,7 +45,6 @@ import org.apache.impala.thrift.TPrioritizeLoadResponse;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TResultRow;
import org.apache.impala.thrift.TStatus;
-import org.apache.impala.thrift.TStartupOptions;
import org.apache.impala.thrift.TSymbolLookupParams;
import org.apache.impala.thrift.TSymbolLookupResult;
import org.apache.impala.thrift.TTable;
@@ -84,9 +83,6 @@ public class FeSupport {
// using Java Thrift bindings.
public native static byte[] NativePrioritizeLoad(byte[] thriftReq);
- // Return select BE startup options as a serialized TStartupOptions
- public native static byte[] NativeGetStartupOptions();
-
/**
* Locally caches the jar at the specified HDFS location.
*
@@ -263,20 +259,6 @@ public class FeSupport {
}
}
- public static TStartupOptions GetStartupOptions() throws InternalException {
- try {
- byte[] result = NativeGetStartupOptions();
- Preconditions.checkNotNull(result);
- TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
- TStartupOptions options = new TStartupOptions();
- deserializer.deserialize(options, result);
- return options;
- } catch (TException e) {
- throw new InternalException("Error retrieving startup options: " + e.getMessage(),
- e);
- }
- }
-
/**
* This function should only be called explicitly by the FeSupport to ensure that
* native functions are loaded.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/JniCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniCatalog.java b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
index 7993191..db9ba63 100644
--- a/fe/src/main/java/org/apache/impala/service/JniCatalog.java
+++ b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
@@ -47,6 +47,7 @@ import org.apache.impala.thrift.TResetMetadataRequest;
import org.apache.impala.thrift.TSentryAdminCheckRequest;
import org.apache.impala.thrift.TUniqueId;
import org.apache.impala.thrift.TUpdateCatalogRequest;
+import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.util.GlogAppender;
import org.apache.impala.util.PatternMatcher;
import org.apache.thrift.TException;
@@ -78,29 +79,30 @@ public class JniCatalog {
return new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
}
- public JniCatalog(boolean loadInBackground, int numMetadataLoadingThreads,
- String sentryServiceConfig, int impalaLogLevel, int otherLogLevel,
- boolean allowAuthToLocal, String kerberosPrincipal, String localLibraryPath,
- int kuduOperationTimeoutMs) throws InternalException {
- BackendConfig.setAuthToLocal(allowAuthToLocal);
- BackendConfig.setKuduClientTimeoutMs(kuduOperationTimeoutMs);
- Preconditions.checkArgument(numMetadataLoadingThreads > 0);
+ public JniCatalog(byte[] thriftBackendConfig) throws InternalException,
+ ImpalaException, TException {
+ TBackendGflags cfg = new TBackendGflags();
+ JniUtil.deserializeThrift(protocolFactory_, cfg, thriftBackendConfig);
+
+ BackendConfig.create(cfg);
+
+ Preconditions.checkArgument(cfg.num_metadata_loading_threads > 0);
// This trick saves having to pass a TLogLevel enum, which is an object and more
// complex to pass through JNI.
- GlogAppender.Install(TLogLevel.values()[impalaLogLevel],
- TLogLevel.values()[otherLogLevel]);
+ GlogAppender.Install(TLogLevel.values()[cfg.impala_log_lvl],
+ TLogLevel.values()[cfg.non_impala_java_vlog]);
// Check if the Sentry Service is configured. If so, create a configuration object.
SentryConfig sentryConfig = null;
- if (!Strings.isNullOrEmpty(sentryServiceConfig)) {
- sentryConfig = new SentryConfig(sentryServiceConfig);
+ if (!Strings.isNullOrEmpty(cfg.sentry_config)) {
+ sentryConfig = new SentryConfig(cfg.sentry_config);
sentryConfig.loadConfig();
}
LOG.info(JniUtil.getJavaVersion());
- catalog_ = new CatalogServiceCatalog(loadInBackground,
- numMetadataLoadingThreads, sentryConfig, getServiceId(), kerberosPrincipal,
- localLibraryPath);
+ catalog_ = new CatalogServiceCatalog(cfg.load_catalog_in_background,
+ cfg.num_metadata_loading_threads, sentryConfig, getServiceId(), cfg.principal,
+ cfg.local_library_path);
try {
catalog_.reset();
} catch (CatalogException e) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 3ef89e3..5e41af4 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -46,6 +46,7 @@ import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.JniUtil;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TBuildTestDescriptorTableParams;
import org.apache.impala.thrift.TCatalogObject;
import org.apache.impala.thrift.TDatabase;
@@ -81,6 +82,7 @@ import org.apache.impala.thrift.TTableName;
import org.apache.impala.thrift.TUniqueId;
import org.apache.impala.thrift.TUpdateCatalogCacheRequest;
import org.apache.impala.thrift.TUpdateMembershipRequest;
+import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.util.GlogAppender;
import org.apache.impala.util.PatternMatcher;
import org.apache.impala.util.TSessionStateUtil;
@@ -115,20 +117,20 @@ public class JniFrontend {
/**
* Create a new instance of the Jni Frontend.
*/
- public JniFrontend(boolean lazy, String serverName, String authorizationPolicyFile,
- String sentryConfigFile, String authPolicyProviderClass, int impalaLogLevel,
- int otherLogLevel, boolean allowAuthToLocal, String defaultKuduMasterHosts,
- int kuduOperationTimeoutMs)
- throws InternalException {
- BackendConfig.setAuthToLocal(allowAuthToLocal);
- BackendConfig.setKuduClientTimeoutMs(kuduOperationTimeoutMs);
- GlogAppender.Install(TLogLevel.values()[impalaLogLevel],
- TLogLevel.values()[otherLogLevel]);
+ public JniFrontend(byte[] thriftBackendConfig) throws ImpalaException, TException {
+ TBackendGflags cfg = new TBackendGflags();
+ JniUtil.deserializeThrift(protocolFactory_, cfg, thriftBackendConfig);
+
+ BackendConfig.create(cfg);
+
+ GlogAppender.Install(TLogLevel.values()[cfg.impala_log_lvl],
+ TLogLevel.values()[cfg.non_impala_java_vlog]);
// Validate the authorization configuration before initializing the Frontend.
// If there are any configuration problems Impala startup will fail.
- AuthorizationConfig authConfig = new AuthorizationConfig(serverName,
- authorizationPolicyFile, sentryConfigFile, authPolicyProviderClass);
+ AuthorizationConfig authConfig = new AuthorizationConfig(cfg.server_name,
+ cfg.authorization_policy_file, cfg.sentry_config,
+ cfg.authorization_policy_provider_class);
authConfig.validateConfig();
if (authConfig.isEnabled()) {
LOG.info(String.format("Authorization is 'ENABLED' using %s",
@@ -139,7 +141,7 @@ public class JniFrontend {
}
LOG.info(JniUtil.getJavaVersion());
- frontend_ = new Frontend(authConfig, defaultKuduMasterHosts);
+ frontend_ = new Frontend(authConfig, cfg.kudu_master_hosts);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/util/KuduUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/KuduUtil.java b/fe/src/main/java/org/apache/impala/util/KuduUtil.java
index c50bf25..65fae74 100644
--- a/fe/src/main/java/org/apache/impala/util/KuduUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/KuduUtil.java
@@ -54,8 +54,8 @@ public class KuduUtil {
*/
public static KuduClient createKuduClient(String kuduMasters) {
KuduClientBuilder b = new KuduClient.KuduClientBuilder(kuduMasters);
- b.defaultAdminOperationTimeoutMs(BackendConfig.getKuduClientTimeoutMs());
- b.defaultOperationTimeoutMs(BackendConfig.getKuduClientTimeoutMs());
+ b.defaultAdminOperationTimeoutMs(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
+ b.defaultOperationTimeoutMs(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
return b.build();
}