You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2023/03/15 23:09:15 UTC
[impala] branch master updated: IMPALA-11989: ImpalaRuntimeException if Kudu and Impala use different HMS
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 7dbdcfb9a IMPALA-11989: ImpalaRuntimeException if Kudu and Impala use different HMS
7dbdcfb9a is described below
commit 7dbdcfb9afa5928b1406527870ad197c72600cc6
Author: Abhishek Rawat <ab...@gmail.com>
AuthorDate: Wed Dec 21 16:35:35 2022 -0800
IMPALA-11989: ImpalaRuntimeException if Kudu and Impala use different HMS
Added a startup flag 'enable_kudu_impala_hms_check' which is used to
flag an error if Kudu and Impala are using different HMS instance(s).
The default behavior in Impala is not changed and the default value of
the new flag is true.
The check itself could be improved in future to ensure that the backend
store used by HMS is same. For now allowing a config option to disable
the check where it makes sense seems like a good short term solution.
Change-Id: I2b06935b262e4a314ad631167c1cd33319e4b10f
Reviewed-on: http://gerrit.cloudera.org:8080/19605
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/frontend.cc | 2 ++
be/src/util/backend-gflag-util.cc | 2 ++
common/thrift/BackendGflags.thrift | 12 +++++++-----
fe/src/main/java/org/apache/impala/catalog/KuduTable.java | 8 +++++++-
4 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index 8105be7f2..d06c86c8d 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -82,6 +82,8 @@ DEFINE_bool(enable_shell_based_groups_mapping_support, false,
DEFINE_string(kudu_master_hosts, "", "Specifies the default Kudu master(s). The given "
"value should be a comma separated list of hostnames or IP addresses; ports are "
"optional.");
+DEFINE_bool(enable_kudu_impala_hms_check, true, "By default this flag is true. If "
+ "enabled checks that Kudu and Impala are using the same HMS instance(s).");
Frontend::Frontend() {
JniMethodDescriptor methods[] = {
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index d39c5c73b..882b0c77a 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -50,6 +50,7 @@ DECLARE_string(authorized_proxy_group_config);
DECLARE_bool(enable_shell_based_groups_mapping_support);
DECLARE_string(catalog_topic_mode);
DECLARE_string(kudu_master_hosts);
+DECLARE_bool(enable_kudu_impala_hms_check);
DECLARE_string(reserved_words_version);
DECLARE_double(max_filter_error_rate);
DECLARE_int64(min_buffer_size);
@@ -265,6 +266,7 @@ Status PopulateThriftBackendGflags(TBackendGflags& cfg) {
FLAGS_local_catalog_cache_expiration_s);
cfg.__set_server_name(FLAGS_server_name);
cfg.__set_kudu_master_hosts(FLAGS_kudu_master_hosts);
+ cfg.__set_enable_kudu_impala_hms_check(FLAGS_enable_kudu_impala_hms_check);
cfg.__set_read_size(FLAGS_read_size);
cfg.__set_num_metadata_loading_threads(FLAGS_num_metadata_loading_threads);
cfg.__set_max_hdfs_partitions_parallel_load(FLAGS_max_hdfs_partitions_parallel_load);
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index dec1f44ad..5ea3cd107 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -237,13 +237,15 @@ struct TBackendGflags {
103: required string ignored_dir_prefix_list
- 104: required bool enable_reload_events
+ 104: required bool enable_kudu_impala_hms_check
- 105: required TGeospatialLibrary geospatial_library
+ 105: required bool enable_reload_events
- 106: required double query_cpu_count_divisor
+ 106: required TGeospatialLibrary geospatial_library
- 107: required bool processing_cost_use_equal_expr_weight
+ 107: required double query_cpu_count_divisor
- 108: required i64 min_processing_per_thread
+ 108: required bool processing_cost_use_equal_expr_weight
+
+ 109: required i64 min_processing_per_thread
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
index 8f1600a2c..427f2f250 100644
--- a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.impala.analysis.ColumnDef;
import org.apache.impala.analysis.KuduPartitionParam;
import org.apache.impala.common.ImpalaRuntimeException;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TColumn;
import org.apache.impala.thrift.TKuduPartitionByHashParam;
@@ -60,6 +61,10 @@ import com.google.common.collect.Lists;
*/
public class KuduTable extends Table implements FeKuduTable {
+ // Boolean config to enable the check which compares Kudu's and Impala's HMS instances.
+ private static boolean ENABLE_KUDU_IMPALA_HMS_CHECK =
+ BackendConfig.INSTANCE.getBackendCfg().enable_kudu_impala_hms_check;
+
// Alias to the string key that identifies the storage handler for Kudu tables.
public static final String KEY_STORAGE_HANDLER =
hive_metastoreConstants.META_TABLE_STORAGE;
@@ -262,7 +267,8 @@ public class KuduTable extends Table implements FeKuduTable {
throw new ImpalaRuntimeException(
String.format("Error parsing URI: %s", e.getMessage()));
}
- if (hmsHosts != null && kuduHmsHosts != null && hmsHosts.equals(kuduHmsHosts)) {
+ if (hmsHosts != null && kuduHmsHosts != null &&
+ (hmsHosts.equals(kuduHmsHosts) || !ENABLE_KUDU_IMPALA_HMS_CHECK)) {
return true;
}
throw new ImpalaRuntimeException(