You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2023/03/24 16:19:43 UTC
[impala] branch master updated (1cfd41e8b -> 1eb6ed6f2)
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 1cfd41e8b IMPALA-11886: Data cache should support asynchronous writes
new 5e694568d IMPALA-11966: Enable cache_ozone_file_handles by default
new 1eb6ed6f2 IMPALA-12023: Skip resource checking on last executor group set
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/runtime/io/disk-io-mgr.cc | 2 +-
be/src/util/backend-gflag-util.cc | 8 +++++++
bin/impala-config.sh | 24 +++++++++----------
common/thrift/BackendGflags.thrift | 2 ++
.../org/apache/impala/service/BackendConfig.java | 4 ++++
.../java/org/apache/impala/service/Frontend.java | 7 ++++++
tests/custom_cluster/test_executor_groups.py | 27 ++++++++++++++++++++--
tests/custom_cluster/test_hdfs_fd_caching.py | 16 ++++++-------
8 files changed, 66 insertions(+), 24 deletions(-)
[impala] 02/02: IMPALA-12023: Skip resource checking on last executor group set
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1eb6ed6f2901aad6af66c206d446f8aebf4529fe
Author: Riza Suminto <ri...@cloudera.com>
AuthorDate: Thu Mar 23 11:46:37 2023 -0700
IMPALA-12023: Skip resource checking on last executor group set
This patch adds flag skip_resource_checking_on_last_executor_group_set.
If this backend flag is set to true, memory and cpu resource checking
will be skipped when a query is being planned against the last (largest)
executor group set. Setting true will ensure that query will always get
admitted into the last executor group set if it does not fit in any
other group set.
Testing
- Tune test_query_cpu_count_divisor_fraction to run two test case:
cpu within limit, and cpu outside limit.
- Add test_no_skip_resource_checking
Change-Id: I5848e4f67939d3dd2fb105c1ae4ca8e15f2e556f
Reviewed-on: http://gerrit.cloudera.org:8080/19649
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Wenzhe Zhou <wz...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/backend-gflag-util.cc | 8 +++++++
common/thrift/BackendGflags.thrift | 2 ++
.../org/apache/impala/service/BackendConfig.java | 4 ++++
.../java/org/apache/impala/service/Frontend.java | 7 ++++++
tests/custom_cluster/test_executor_groups.py | 27 ++++++++++++++++++++--
5 files changed, 46 insertions(+), 2 deletions(-)
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 882b0c77a..cd8f1dc01 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -222,6 +222,12 @@ DEFINE_int64_hidden(min_processing_per_thread, 10000000,
"number of cores in selected executor group, MT_DOP, or PROCESSING_COST_MIN_THREAD "
"query option. Must be a positive integer. Default to 10M.");
+DEFINE_bool_hidden(skip_resource_checking_on_last_executor_group_set, true,
+ "(Advance) If true, memory and cpu resource checking will be skipped when a query "
+ "is being planned against the last (largest) executor group set. Setting true will "
+ "ensure that query will always get admitted into last executor group set if it does "
+ "not fit in any other group set.");
+
using strings::Substitute;
namespace impala {
@@ -399,6 +405,8 @@ Status PopulateThriftBackendGflags(TBackendGflags& cfg) {
cfg.__set_processing_cost_use_equal_expr_weight(
FLAGS_processing_cost_use_equal_expr_weight);
cfg.__set_min_processing_per_thread(FLAGS_min_processing_per_thread);
+ cfg.__set_skip_resource_checking_on_last_executor_group_set(
+ FLAGS_skip_resource_checking_on_last_executor_group_set);
return Status::OK();
}
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index 5ea3cd107..3f1f03749 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -248,4 +248,6 @@ struct TBackendGflags {
108: required bool processing_cost_use_equal_expr_weight
109: required i64 min_processing_per_thread
+
+ 110: required bool skip_resource_checking_on_last_executor_group_set
}
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index f5daefa7e..820d60a17 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -390,4 +390,8 @@ public class BackendConfig {
public long getMinProcessingPerThread() {
return backendCfg_.min_processing_per_thread;
}
+
+ public boolean isSkipResourceCheckingOnLastExecutorGroupSet() {
+ return backendCfg_.skip_resource_checking_on_last_executor_group_set;
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index e59bf90bc..1952b6e33 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -2122,6 +2122,13 @@ public class Frontend {
matchFound = true;
}
+ if (!matchFound && (i >= num_executor_group_sets - 1)
+ && BackendConfig.INSTANCE.isSkipResourceCheckingOnLastExecutorGroupSet()) {
+ reason = "no executor group set fit. Admit to last executor group set.";
+ addInfoString(groupSetProfile, VERDICT, reason);
+ matchFound = true;
+ }
+
if (matchFound) {
// Set the group name prefix in both the returned query options and
// the query context for non default group setup.
diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py
index 4ae476e81..6c60b703c 100644
--- a/tests/custom_cluster/test_executor_groups.py
+++ b/tests/custom_cluster/test_executor_groups.py
@@ -909,11 +909,34 @@ class TestExecutorGroups(CustomClusterTestSuite):
@pytest.mark.execute_serially
def test_query_cpu_count_divisor_fraction(self):
# Expect to run the query on the large group
- coordinator_test_args = "-query_cpu_count_divisor=0.2 "
+ coordinator_test_args = "-query_cpu_count_divisor=0.03 "
self._setup_three_exec_group_cluster(coordinator_test_args)
+ options = copy.deepcopy(CPU_DOP_OPTIONS)
+ options['MT_DOP'] = '1'
+ self._run_query_and_verify_profile(CPU_TEST_QUERY, options,
+ ["Executor Group: root.large-group", "EffectiveParallelism: 4",
+ "ExecutorGroupsConsidered: 3", "CpuAsk: 134",
+ "Verdict: Match"])
+
+ # Expect that a query still admitted to last group even if
+ # its resource requirement exceed the limit on that last executor group.
self._run_query_and_verify_profile(CPU_TEST_QUERY, CPU_DOP_OPTIONS,
["Executor Group: root.large-group", "EffectiveParallelism: 7",
- "ExecutorGroupsConsidered: 3"])
+ "ExecutorGroupsConsidered: 3", "CpuAsk: 234",
+ "Verdict: no executor group set fit. Admit to last executor group set."])
+ self.client.close()
+
+ @pytest.mark.execute_serially
+ def test_no_skip_resource_checking(self):
+ """This test check that executor group limit is enforced if
+ skip_resource_checking_on_last_executor_group_set=false."""
+ coordinator_test_args = ("-query_cpu_count_divisor=0.03 "
+ "-skip_resource_checking_on_last_executor_group_set=false ")
+ self._setup_three_exec_group_cluster(coordinator_test_args)
+ self.client.set_configuration(CPU_DOP_OPTIONS)
+ result = self.execute_query_expect_failure(self.client, CPU_TEST_QUERY)
+ assert ("AnalysisException: The query does not fit largest executor group sets. "
+ "Reason: not enough cpu cores (require=234, max=192).") in str(result)
self.client.close()
@pytest.mark.execute_serially
[impala] 01/02: IMPALA-11966: Enable cache_ozone_file_handles by default
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5e694568d5e837e6f6648bed573bb3b60c7d5a92
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Thu Mar 2 15:47:48 2023 -0800
IMPALA-11966: Enable cache_ozone_file_handles by default
Updates Ozone dependency to 1.3.0 to address HDDS-7135 and enables
cache_ozone_file_handles by default for a ~10% improvement on TPC-DS
query time.
Updates the Ozone CDP dependency for HDDS-8095. Fix for it will be
available in Ozone 1.4.0, so testing with TDE currently requires the CDP
build.
Testing:
- ran backend, e2e, and custom cluster test suites with Ozone
Change-Id: Icc66551f9b87af785a1c30b516ac39f4640638fe
Reviewed-on: http://gerrit.cloudera.org:8080/19573
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/runtime/io/disk-io-mgr.cc | 2 +-
bin/impala-config.sh | 24 ++++++++++++------------
tests/custom_cluster/test_hdfs_fd_caching.py | 16 +++++++---------
3 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/be/src/runtime/io/disk-io-mgr.cc b/be/src/runtime/io/disk-io-mgr.cc
index 29babb3e0..fae86e948 100644
--- a/be/src/runtime/io/disk-io-mgr.cc
+++ b/be/src/runtime/io/disk-io-mgr.cc
@@ -199,7 +199,7 @@ DEFINE_bool(cache_s3_file_handles, true, "Enable the file handle cache for "
DEFINE_bool(cache_abfs_file_handles, true, "Enable the file handle cache for "
"ABFS files.");
-DEFINE_bool(cache_ozone_file_handles, false, "Enable the file handle cache for Ozone "
+DEFINE_bool(cache_ozone_file_handles, true, "Enable the file handle cache for Ozone "
"files.");
DECLARE_int64(min_buffer_size);
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 2470ebb15..acfd43210 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -213,26 +213,26 @@ fi
: ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com}
export IMPALA_TOOLCHAIN_HOST
-export CDP_BUILD_NUMBER=38235009
+export CDP_BUILD_NUMBER=39127492
export CDP_MAVEN_REPOSITORY=\
"https://${IMPALA_TOOLCHAIN_HOST}/build/cdp_components/${CDP_BUILD_NUMBER}/maven"
-export CDP_AVRO_JAVA_VERSION=1.8.2.7.2.17.0-127
-export CDP_HADOOP_VERSION=3.1.1.7.2.17.0-127
-export CDP_HBASE_VERSION=2.4.6.7.2.17.0-127
-export CDP_HIVE_VERSION=3.1.3000.7.2.17.0-127
-export CDP_ICEBERG_VERSION=1.1.0.7.2.17.0-127
-export CDP_KNOX_VERSION=1.3.0.7.2.17.0-127
-export CDP_OZONE_VERSION=1.3.0.7.2.17.0-127
-export CDP_PARQUET_VERSION=1.10.99.7.2.17.0-127
-export CDP_RANGER_VERSION=2.3.0.7.2.17.0-127
-export CDP_TEZ_VERSION=0.9.1.7.2.17.0-127
+export CDP_AVRO_JAVA_VERSION=1.8.2.7.2.17.0-160
+export CDP_HADOOP_VERSION=3.1.1.7.2.17.0-160
+export CDP_HBASE_VERSION=2.4.6.7.2.17.0-160
+export CDP_HIVE_VERSION=3.1.3000.7.2.17.0-160
+export CDP_ICEBERG_VERSION=1.1.0.7.2.17.0-160
+export CDP_KNOX_VERSION=1.3.0.7.2.17.0-160
+export CDP_OZONE_VERSION=1.3.0.7.2.17.0-160
+export CDP_PARQUET_VERSION=1.10.99.7.2.17.0-160
+export CDP_RANGER_VERSION=2.3.0.7.2.17.0-160
+export CDP_TEZ_VERSION=0.9.1.7.2.17.0-160
# Ref: https://infra.apache.org/release-download-pages.html#closer
: ${APACHE_MIRROR:="https://www.apache.org/dyn/closer.cgi"}
export APACHE_MIRROR
export APACHE_HIVE_VERSION=3.1.3
export APACHE_HIVE_STORAGE_API_VERSION=2.7.0
-export APACHE_OZONE_VERSION=1.2.1
+export APACHE_OZONE_VERSION=1.3.0
export ARCH_NAME=$(uname -p)
diff --git a/tests/custom_cluster/test_hdfs_fd_caching.py b/tests/custom_cluster/test_hdfs_fd_caching.py
index b5e5db5e8..9cb6936a2 100644
--- a/tests/custom_cluster/test_hdfs_fd_caching.py
+++ b/tests/custom_cluster/test_hdfs_fd_caching.py
@@ -125,8 +125,7 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--max_cached_file_handles=16"
- " --unused_file_handle_timeout_sec=18446744073709551600"
- " --cache_ozone_file_handles=true",
+ " --unused_file_handle_timeout_sec=18446744073709551600",
catalogd_args="--load_catalog_in_background=false")
def test_caching_enabled(self, vector):
"""
@@ -146,8 +145,7 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
- impalad_args="--max_cached_file_handles=16 --unused_file_handle_timeout_sec=5"
- " --cache_ozone_file_handles=true",
+ impalad_args="--max_cached_file_handles=16 --unused_file_handle_timeout_sec=5",
catalogd_args="--load_catalog_in_background=false")
def test_caching_with_eviction(self, vector):
"""Test of the HDFS file handle cache with unused file handle eviction enabled"""
@@ -162,7 +160,7 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
- impalad_args="--max_cached_file_handles=0 --cache_ozone_file_handles=true",
+ impalad_args="--max_cached_file_handles=0",
catalogd_args="--load_catalog_in_background=false")
def test_caching_disabled_by_param(self, vector):
"""Test that the HDFS file handle cache is disabled when the parameter is zero"""
@@ -173,7 +171,8 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--cache_remote_file_handles=false --cache_s3_file_handles=false "
- "--cache_abfs_file_handles=false --hostname=" + get_external_ip(),
+ "--cache_abfs_file_handles=false --cache_ozone_file_handles=false "
+ "--hostname=" + get_external_ip(),
catalogd_args="--load_catalog_in_background=false")
def test_remote_caching_disabled_by_param(self, vector):
"""Test that the file handle cache is disabled for remote files when disabled"""
@@ -183,8 +182,7 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
- impalad_args="--max_cached_file_handles=0 --cache_ozone_file_handles=true "
- "--hostname=" + get_external_ip(),
+ impalad_args="--max_cached_file_handles=0 --hostname=" + get_external_ip(),
catalogd_args="--load_catalog_in_background=false")
def test_remote_caching_disabled_by_global_param(self, vector):
"""Test that the file handle cache is disabled for remote files when all caching is
@@ -196,7 +194,7 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--max_cached_file_handles=16 --unused_file_handle_timeout_sec=5 "
- "--always_use_data_cache=true --cache_ozone_file_handles=true",
+ "--always_use_data_cache=true",
start_args="--data_cache_dir=/tmp --data_cache_size=500MB",
catalogd_args="--load_catalog_in_background=false")
def test_no_fd_caching_on_cached_data(self, vector):