You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2025/06/25 03:39:42 UTC
(impala) branch master updated (aba3a705a -> d36df0eb8)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from aba3a705a IMPALA-13982: Support regular views for Calcite planner in Impala
new 1d0b2ef0c IMPALA-14164: Fix timeout for fragments in flight in TestScratchDir
new d36df0eb8 IMPALA-14163: Raise test_pool_config_change_while_queued MEM_LIMIT
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
tests/common/resource_pool_config.py | 5 +++--
tests/custom_cluster/test_admission_controller.py | 1 -
tests/custom_cluster/test_scratch_disk.py | 19 -------------------
3 files changed, 3 insertions(+), 22 deletions(-)
(impala) 01/02: IMPALA-14164: Fix timeout for fragments in flight in TestScratchDir
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1d0b2ef0c593145c476d52610c0f4ec2c69c8be7
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Mon Jun 23 20:31:00 2025 -0700
IMPALA-14164: Fix timeout for fragments in flight in TestScratchDir
On release builds, some tests in TestScratchDir have started hitting
a timeout waiting for num-fragments-in-flight to reach 2. The code
to wait for the metric sleeps one second between samples. If one of
the query fragments starts and finishes during that second, the test
will never see a sample containing two in-flight fragments. This
happens on release builds because they are faster and more likely to
complete within that second.
This removes the code that waits for num-fragments-in-flight. All the
tests have subsequent calls waiting for the scratch usage to reach a
certain value. This will properly wait for the fragment to start up
on its own. The num-fragments-in-flight wait doesn't add anything.
Testing:
- Ran custom_cluster/test_scratch_disk.py multiple times with a
release build
Change-Id: Ic8c573affc033056ba465c42bd420d5c1d3ba15c
Reviewed-on: http://gerrit.cloudera.org:8080/23081
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/custom_cluster/test_scratch_disk.py | 19 -------------------
1 file changed, 19 deletions(-)
diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py
index 87dc3baf8..3a57f5a1e 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -30,7 +30,6 @@ import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import SkipIf
from tests.util.hdfs_util import NAMENODE
-from tests.verifiers.metric_verifier import MetricVerifier
class TestScratchDir(CustomClusterTestSuite):
@@ -234,8 +233,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
handle = self.execute_query_async_using_client(client, self.spill_query, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
for i in range(5):
impalad.service.wait_for_metric_value(
'tmp-file-mgr.scratch-space-bytes-used.dir-' + str(i), 1, allow_greater=True)
@@ -264,8 +261,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
handle = self.execute_query_async_using_client(client, self.spill_query, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
# dir1 and dir3 have highest priority and will be used as scratch disk.
impalad.service.wait_for_metric_value(
'tmp-file-mgr.scratch-space-bytes-used.dir-1', 1, allow_greater=True)
@@ -333,8 +328,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
handle = self.execute_query_async_using_client(client, self.spill_query, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
# Dir0 is the remote directory.
impalad.service.wait_for_metric_value(
'tmp-file-mgr.scratch-space-bytes-used.dir-0', 1, allow_greater=True)
@@ -364,8 +357,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
handle = self.execute_query_async_using_client(client, self.spill_query, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
# Local directory always ranks before the remote one, so dir0 is the local directory.
# Only spill to dir0 because it has enough space for the spilling.
impalad.service.wait_for_metric_value(
@@ -398,8 +389,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
handle = self.execute_query_async_using_client(client, self.spill_query, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
# Local directory always ranks before the remote one, so dir0 is the local directory.
# The query spills to both dir0 and dir1. By default the remote file is 16MB each,
# so the value of metrics1 should be at least one file size.
@@ -431,8 +420,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
handle = self.execute_query_async_using_client(client, self.spill_query, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
# The query spills to the remote directories and creates remote files,
# so that the size is bigger than 0, and be integer times of remote file size.
impalad.service.wait_for_metric_value(
@@ -515,8 +502,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
handle = self.execute_query_async_using_client(client, self.spill_query, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
results = client.fetch(self.spill_query, handle)
assert results.success
metrics0 = self.get_metric(
@@ -544,8 +529,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
self.execute_query_async_using_client(client, self.spill_query_big_table, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
# Dir0 is the remote directory.
impalad.service.wait_for_metric_value(
'tmp-file-mgr.scratch-space-bytes-used.dir-0', 1, allow_greater=True)
@@ -585,8 +568,6 @@ class TestScratchDir(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
client = impalad.service.create_hs2_client()
self.execute_query_async_using_client(client, self.spill_query_big_table, vector)
- verifier = MetricVerifier(impalad.service)
- verifier.wait_for_metric("impala-server.num-fragments-in-flight", 2)
# Dir0 is the remote directory.
impalad.service.wait_for_metric_value(
'tmp-file-mgr.scratch-space-bytes-used.dir-0', 1, allow_greater=True)
(impala) 02/02: IMPALA-14163: Raise test_pool_config_change_while_queued MEM_LIMIT
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit d36df0eb8847d527eeb4c62feb8a78a322605a3e
Author: Riza Suminto <ri...@cloudera.com>
AuthorDate: Mon Jun 23 12:58:47 2025 -0700
IMPALA-14163: Raise test_pool_config_change_while_queued MEM_LIMIT
test_pool_config_change_while_queued hit timeout at
TestAdmissionControllerWithACService. When running this test locally, we
notice that some trigger query ("select 'wait_for_config_change'")
passed when it is expected to be rejected (hit EXCEPTION during
admission).
This patch increase the MEM_LIMIT higher to 128GB to ensure rejection.
It also add wait_for_admission_control() that should immediately return
once trigger query hit exception. Removed redundant
"set enable_trivial_query_for_admission=false" query in
test_pool_config_change_while_queued.
Testing:
- Loop the test couple times and confirm that all trigger query
executions hit exception.
Change-Id: Iee808d0fc92308604ed0ee27dde795e9aa69eb5d
Reviewed-on: http://gerrit.cloudera.org:8080/23072
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/common/resource_pool_config.py | 5 +++--
tests/custom_cluster/test_admission_controller.py | 1 -
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/tests/common/resource_pool_config.py b/tests/common/resource_pool_config.py
index 0c53bbdf0..e2ace0115 100644
--- a/tests/common/resource_pool_config.py
+++ b/tests/common/resource_pool_config.py
@@ -65,12 +65,13 @@ class ResourcePoolConfig(object):
client.set_configuration_option('request_pool', pool_name)
# set mem_limit to something above the proc limit so that the query always gets
# rejected.
- client.set_configuration_option('mem_limit', '20G')
+ client.set_configuration_option('mem_limit', '128G')
+ client.set_configuration_option("enable_trivial_query_for_admission", "false")
metric_key = "admission-controller.{0}.root.{1}".format(metric_str, pool_name)
start_time = time()
while (time() - start_time < timeout):
- client.execute("set enable_trivial_query_for_admission=false")
handle = client.execute_async("select 'wait_for_config_change'")
+ client.wait_for_admission_control(handle, timeout)
client.close_query(handle)
current_val = str(self.ac_service.get_metric_value(metric_key))
if current_val == target_val:
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 92e6d5eaf..358424012 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -1472,7 +1472,6 @@ class TestAdmissionController(TestAdmissionControllerBase):
self.client.wait_for_admission_control(sleep_query_handle)
self._wait_for_change_to_profile(sleep_query_handle,
"Admission result: Admitted immediately")
- self.client.execute("set enable_trivial_query_for_admission=false")
queued_query_handle = self.client.execute_async("select 2")
self._wait_for_change_to_profile(queued_query_handle, "Admission result: Queued")