You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues-all@impala.apache.org by "Quanlong Huang (Jira)" <ji...@apache.org> on 2026/04/24 05:49:00 UTC
[jira] [Commented] (IMPALA-14696) TestSessionExpiration.test_closing_idle_connection is flaky
[ https://issues.apache.org/jira/browse/IMPALA-14696?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18075894#comment-18075894 ]
Quanlong Huang commented on IMPALA-14696:
-----------------------------------------
Saw a variant of this failure. Might be the same cause.
{noformat}
custom_cluster/test_session_expiration.py:185: in test_closing_idle_connection
assert num_hs2_connections + 1 == impalad.service.get_metric_value(
E assert 1 == 2
E +1
E -2
client = <tests.common.impala_connection.ImpylaHS2Connection object at 0x7f8e84b827f0>
impalad = <ImpaladProcess PID: 2256419 (/data/jenkins/workspace/impala-cdw-master-exhaustive-release/repos/Impala/be/build/lates..._io_threads=1 -num_s3_io_threads=1 -num_s3_file_oper_io_threads=1 -num_sfs_io_threads=1 -geospatial_library=HIVE_ESRI)>
num_connections = 0
num_connections_metrics_name = 'impala.thrift-server.hiveserver2-frontend.connections-in-use'
num_expired = 1
num_hs2_connections = 0
protocol = 'hiveserver2'
self = <tests.custom_cluster.test_session_expiration.TestSessionExpiration object at 0x7f8e85d9f700>
sock = <socket.socket fd=15, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 52792), raddr=('127.0.0.1', 21050)>{noformat}
> TestSessionExpiration.test_closing_idle_connection is flaky
> -----------------------------------------------------------
>
> Key: IMPALA-14696
> URL: https://issues.apache.org/jira/browse/IMPALA-14696
> Project: IMPALA
> Issue Type: Bug
> Reporter: Dániel Gábor Vankó
> Assignee: Dániel Gábor Vankó
> Priority: Major
> Labels: broken-build, flaky, flaky-test
>
> Saw in some test that test_closing_idle_connection fails with the following:
> h2. Error Message
> {noformat}
> AssertionError: Metric impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach value 0 in 5s. Actual value was '1'.
> Dumping debug webpages in JSON format...
> Dumped memz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/memz.json
> Dumped metrics JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/metrics.json
> Dumped queries JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/queries.json
> Dumped sessions JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/sessions.json
> Dumped threadz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/threadz.json
> Dumped rpcz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json
> Dumping minidumps for impalads/catalogds...
> Dumped minidump for Impalad PID 250362
> Dumped minidump for Catalogd PID 250268{noformat}
> h2.
> Stacktrace
> {noformat}
> custom_cluster/test_session_expiration.py:171: in test_closing_idle_connection
> impalad.service.wait_for_metric_value(num_connections_metrics_name,
> client = <tests.common.impala_connection.ImpylaHS2Connection object at 0xffff079248b0>
> impalad = <ImpaladProcess PID: 250362 (/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/be/build/latest/serv..._io_threads=1 -num_s3_io_threads=1 -num_s3_file_oper_io_threads=1 -num_sfs_io_threads=1 -geospatial_library=HIVE_ESRI)>
> num_connections = 0
> num_connections_metrics_name = 'impala.thrift-server.hiveserver2-frontend.connections-in-use'
> num_expired = 1
> protocol = 'hiveserver2'
> self = <tests.custom_cluster.test_session_expiration.TestSessionExpiration object at 0xffff072ad760>
> common/impala_service.py:164: in wait_for_metric_value
> self.__metric_timeout_assert(metric_name, expected_value, timeout, value)
> allow_greater = False
> expected_value = 0
> interval = 1
> metric_name = 'impala.thrift-server.hiveserver2-frontend.connections-in-use'
> self = <tests.common.impala_service.ImpaladService object at 0xffff0792ae20>
> start_time = 1769023935.4829047
> timeout = 5
> total_wait = 5.0343897342681885
> value = 1
> common/impala_service.py:251: in __metric_timeout_assert
> assert 0, assert_string
> E AssertionError: Metric impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach value 0 in 5s. Actual value was '1'.
> E Dumping debug webpages in JSON format...
> E Dumped memz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/memz.json
> E Dumped metrics JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/metrics.json
> E Dumped queries JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/queries.json
> E Dumped sessions JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/sessions.json
> E Dumped threadz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/threadz.json
> E Dumped rpcz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json
> E Dumping minidumps for impalads/catalogds...
> E Dumped minidump for Impalad PID 250362
> E Dumped minidump for Catalogd PID 250268
> actual_value = 1
> assert_string = "Metric impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach value 0 in 5s. Actual value was '1'...minidumps for impalads/catalogds...\nDumped minidump for Impalad PID 250362\nDumped minidump for Catalogd PID 250268\n"
> catalogd_pids = ['250268']
> datetime_string = '20260121_11:32:20'
> debug_page = 'rpcz'
> debug_pages = ['memz', 'metrics', 'queries', 'sessions', 'threadz', 'rpcz']
> diag_dir = '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/metric_timeout_diags_20260121_11:32:20'
> expected_value = 0
> impala_home = '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala'
> impalad_pids = ['250362']
> json_diag_string = 'Dumping debug webpages in JSON format...\nDumped memz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:2...32:20/json/threadz.json\nDumped rpcz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json\n'
> json_dir = '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/metric_timeout_diags_20260121_11:32:20/json'
> json_filename = '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json'
> json_filename_rewritten = '$IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json'
> log_dir = '/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs'
> metric_name = 'impala.thrift-server.hiveserver2-frontend.connections-in-use'
> minidump_diag_string = 'Dumping minidumps for impalads/catalogds...\nDumped minidump for Impalad PID 250362\nDumped minidump for Catalogd PID 250268\n'
> pid = '250268'
> self = <tests.common.impala_service.ImpaladService object at 0xffff0792ae20>
> timeout = 5{noformat}
> h2. Standard Error
> {noformat}
> --------------------------------- Captured Err ---------------------------------
> -- 2026-01-21 11:31:44,397 INFO MainThread: Starting cluster with command: /data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/bin/start-impala-cluster.py '--state_store_args=--statestore_update_frequency_ms=50 --statestore_priority_update_frequency_ms=50 --statestore_heartbeat_frequency_ms=50' --cluster_size=1 --num_coordinators=1 --log_dir=/data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/custom_cluster_tests/TestSessionExpiration/test_closing_idle_connection --log_level=1 '--impalad_args=--idle_session_timeout=10 --idle_client_poll_period_s=1 ' --impalad_args=--default_query_options=
> 11:31:44 MainThread: Found 0 impalad/0 statestored/0 catalogd process(es)
> 11:31:44 MainThread: Starting State Store logging to /data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/custom_cluster_tests/TestSessionExpiration/test_closing_idle_connection/statestored.INFO
> 11:31:45 MainThread: Starting Catalog Service logging to /data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/custom_cluster_tests/TestSessionExpiration/test_closing_idle_connection/catalogd.INFO
> 11:31:47 MainThread: Starting Impala Daemon logging to /data/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/logs/custom_cluster_tests/TestSessionExpiration/test_closing_idle_connection/impalad.INFO
> 11:31:49 MainThread: Found 1 impalad/1 statestored/1 catalogd process(es)
> 11:31:49 MainThread: Waiting for Impalad webserver port 25000
> 11:31:49 MainThread: Waiting for Impalad webserver port 25000
> 11:31:50 MainThread: Waiting for Impalad webserver port 25000
> 11:31:50 MainThread: Waiting for Impalad webserver port 25000
> 11:31:52 MainThread: Waiting for coordinator client services - hs2 port: 21050 hs2-http port: 28000 beeswax port: 21000
> 11:31:52 MainThread: Getting num_known_live_backends from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> 11:31:52 MainThread: num_known_live_backends has reached value: 1
> 11:31:52 MainThread: Total wait: 3.65s
> 11:31:52 MainThread: Actual log file names:
> 11:31:52 MainThread: statestored.INFO -> statestored.impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com.jenkins.log.INFO.20260121-113144.250206
> 11:31:52 MainThread: catalogd.INFO -> catalogd.impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com.jenkins.log.INFO.20260121-113146.250268
> 11:31:52 MainThread: impalad.INFO -> impalad.impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com.jenkins.log.INFO.20260121-113147.250362
> 11:31:52 MainThread: Impala Cluster Running with 1 nodes (1 coordinators, 1 executors).
> -- 2026-01-21 11:31:52,772 DEBUG MainThread: Found 1 impalad/1 statestored/1 catalogd process(es)
> -- 2026-01-21 11:31:52,772 INFO MainThread: Getting metric: statestore.live-backends from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25010
> -- 2026-01-21 11:31:52,778 INFO MainThread: Metric 'statestore.live-backends' has reached desired value: 2. total_wait: 0s
> -- 2026-01-21 11:31:52,778 DEBUG MainThread: Getting num_known_live_backends from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:52,782 INFO MainThread: num_known_live_backends has reached value: 1
> -- 2026-01-21 11:31:52,783 INFO MainThread: beeswax:
> set client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,783 INFO MainThread: beeswax: connected to impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000 with beeswax
> -- 2026-01-21 11:31:52,783 INFO MainThread: hs2:
> set client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,783 INFO MainThread: hs2: connected to impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050 with impyla hs2
> -- 2026-01-21 11:31:52,783 INFO MainThread: hs2-http:
> set client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,785 INFO MainThread: hs2-http: connected to impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:28000 with impyla hs2-http
> -- 2026-01-21 11:31:52,785 INFO MainThread: hs2-feng:
> set client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,785 INFO MainThread: hs2-feng: connected to impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:11050 with impyla hs2-feng
> -- 2026-01-21 11:31:52,788 INFO MainThread: beeswax: closing beeswax connection to: impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000
> -- 2026-01-21 11:31:52,788 INFO MainThread: hs2: closing 1 sync and 0 async hs2 connections to: impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050
> -- 2026-01-21 11:31:52,788 INFO MainThread: hs2-http: closing 1 sync and 0 async hs2-http connections to: impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:28000
> -- 2026-01-21 11:31:52,788 INFO MainThread: hs2-feng: closing 1 sync and 0 async hs2-feng connections to: impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:11050
> -- 2026-01-21 11:31:52,809 WARNING MainThread: beeswax protocol is deprecated.
> -- 2026-01-21 11:31:52,809 INFO MainThread: beeswax:
> set client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:31:52,809 INFO MainThread: beeswax: connected to impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000 with beeswax
> -- 2026-01-21 11:31:52,809 INFO MainThread: beeswax: executing against impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000
> select 1;
> -- 2026-01-21 11:31:53,191 INFO MainThread: Started query df4d8a404151a8b0:a0da1a1000000000
> -- 2026-01-21 11:31:53,247 INFO MainThread: Getting metric: impala.thrift-server.beeswax-frontend.connections-in-use from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:53,253 INFO MainThread: Metric 'impala.thrift-server.beeswax-frontend.connections-in-use' has reached desired value: 1. total_wait: 0s
> -- 2026-01-21 11:31:53,254 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:53,259 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 0s
> -- 2026-01-21 11:31:53,259 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:54,260 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:54,266 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 1.0065803527832031s
> -- 2026-01-21 11:31:54,267 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:55,267 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:55,273 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 2.0133278369903564s
> -- 2026-01-21 11:31:55,273 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:56,274 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:56,280 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 3.0207297801971436s
> -- 2026-01-21 11:31:56,281 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:57,282 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:57,288 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 4.028151035308838s
> -- 2026-01-21 11:31:57,288 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:58,289 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:58,295 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 5.03555703163147s
> -- 2026-01-21 11:31:58,295 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:31:59,296 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:31:59,302 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 6.0421106815338135s
> -- 2026-01-21 11:31:59,302 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:00,303 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:00,309 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 7.049506664276123s
> -- 2026-01-21 11:32:00,309 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:01,310 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:01,317 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 8.056854486465454s
> -- 2026-01-21 11:32:01,317 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:02,318 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:02,324 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 9.06404423713684s
> -- 2026-01-21 11:32:02,324 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:03,325 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:03,331 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=1. Current value: 0. total_wait: 10.07157278060913s
> -- 2026-01-21 11:32:03,331 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:04,333 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:04,339 INFO MainThread: Metric 'impala-server.num-sessions-expired' has reached desired value: 1. total_wait: 11.078944683074951s
> -- 2026-01-21 11:32:04,339 INFO MainThread: Getting metric: impala.thrift-server.beeswax-frontend.connections-in-use from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:04,344 INFO MainThread: Metric 'impala.thrift-server.beeswax-frontend.connections-in-use' has reached desired value: 0. total_wait: 0s
> -- 2026-01-21 11:32:04,344 INFO MainThread: beeswax: closing beeswax connection to: impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21000
> -- 2026-01-21 11:32:04,354 INFO MainThread: hs2:
> set client_identifier=custom_cluster/test_session_expiration.py::TestSessionExpiration::test_closing_idle_connection;
> -- 2026-01-21 11:32:04,355 INFO MainThread: hs2: connected to impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050 with impyla hs2
> -- 2026-01-21 11:32:04,356 INFO MainThread: hs2: executing against Impala at impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050. session: 204b4c0587e2d9c0:0c9b2506804d61ac main_cursor: True user: None
> select 1;
> -- 2026-01-21 11:32:04,383 INFO MainThread: 5d43097c66b5e745:cad5462c00000000: query started
> -- 2026-01-21 11:32:04,386 INFO MainThread: 5d43097c66b5e745:cad5462c00000000: getting log for operation
> -- 2026-01-21 11:32:04,387 INFO MainThread: 5d43097c66b5e745:cad5462c00000000: getting runtime profile operation
> -- 2026-01-21 11:32:04,388 INFO MainThread: 5d43097c66b5e745:cad5462c00000000: closing query for operation
> -- 2026-01-21 11:32:04,389 INFO MainThread: Getting metric: impala.thrift-server.hiveserver2-frontend.connections-in-use from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:04,394 INFO MainThread: Could not connect to ('::1', 21050, 0, 0)
> Traceback (most recent call last):
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/urllib3/connectionpool.py", line 377, in _make_request
> httplib_response = conn.getresponse(buffering=True)
> TypeError: getresponse() got an unexpected keyword argument 'buffering'
> During handling of the above exception, another exception occurred:
> Traceback (most recent call last):
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/thrift/transport/TSocket.py", line 137, in open
> handle.connect(sockaddr)
> ConnectionRefusedError: [Errno 111] Connection refused
> -- 2026-01-21 11:32:04,397 INFO MainThread: Metric 'impala.thrift-server.hiveserver2-frontend.connections-in-use' has reached desired value: 1. total_wait: 0s
> -- 2026-01-21 11:32:04,397 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:04,402 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 0s
> -- 2026-01-21 11:32:04,402 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:05,403 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:05,409 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 1.0054309368133545s
> -- 2026-01-21 11:32:05,409 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:06,410 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:06,416 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 2.0127756595611572s
> -- 2026-01-21 11:32:06,416 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:07,417 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:07,423 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 3.0200905799865723s
> -- 2026-01-21 11:32:07,423 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:08,425 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:08,431 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 4.027472257614136s
> -- 2026-01-21 11:32:08,431 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:09,432 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:09,438 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 5.034852981567383s
> -- 2026-01-21 11:32:09,438 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:10,439 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:10,445 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 6.042134046554565s
> -- 2026-01-21 11:32:10,445 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:11,447 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:11,453 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 7.049464464187622s
> -- 2026-01-21 11:32:11,453 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:12,454 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:12,460 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 8.057080030441284s
> -- 2026-01-21 11:32:12,460 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:13,462 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:13,468 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 9.064432859420776s
> -- 2026-01-21 11:32:13,468 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:14,469 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:14,475 INFO MainThread: Waiting for metric value 'impala-server.num-sessions-expired'=2. Current value: 1. total_wait: 10.071696281433105s
> -- 2026-01-21 11:32:14,475 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:15,476 INFO MainThread: Getting metric: impala-server.num-sessions-expired from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:15,482 INFO MainThread: Metric 'impala-server.num-sessions-expired' has reached desired value: 2. total_wait: 11.079020500183105s
> -- 2026-01-21 11:32:15,482 INFO MainThread: Getting metric: impala.thrift-server.hiveserver2-frontend.connections-in-use from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:15,488 INFO MainThread: Waiting for metric value 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current value: 1. total_wait: 0s
> -- 2026-01-21 11:32:15,488 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:16,489 INFO MainThread: Getting metric: impala.thrift-server.hiveserver2-frontend.connections-in-use from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:16,495 INFO MainThread: Waiting for metric value 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current value: 1. total_wait: 1.006498098373413s
> -- 2026-01-21 11:32:16,495 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:17,496 INFO MainThread: Getting metric: impala.thrift-server.hiveserver2-frontend.connections-in-use from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:17,502 INFO MainThread: Waiting for metric value 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current value: 1. total_wait: 2.0138399600982666s
> -- 2026-01-21 11:32:17,502 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:18,503 INFO MainThread: Getting metric: impala.thrift-server.hiveserver2-frontend.connections-in-use from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:18,509 INFO MainThread: Waiting for metric value 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current value: 1. total_wait: 3.0201923847198486s
> -- 2026-01-21 11:32:18,509 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:19,509 INFO MainThread: Getting metric: impala.thrift-server.hiveserver2-frontend.connections-in-use from impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:25000
> -- 2026-01-21 11:32:19,516 INFO MainThread: Waiting for metric value 'impala.thrift-server.hiveserver2-frontend.connections-in-use'=0. Current value: 1. total_wait: 4.02702784538269s
> -- 2026-01-21 11:32:19,516 INFO MainThread: Sleeping 1s before next retry.
> -- 2026-01-21 11:32:20,517 INFO MainThread: Metric impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach value 0 in 5s. Actual value was '1'. total_wait: 5.0343897342681885s. Failing...
> -- 2026-01-21 11:32:50,693 INFO MainThread: hs2: closing 1 sync and 0 async hs2 connections to: impala-ec2-rhel92-m7g-4xlarge-ondemand-1319.vpc.cloudera.com:21050
> -- 2026-01-21 11:32:50,694 ERROR MainThread: Failed to open transport (tries_left=3)
> Traceback (most recent call last):
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/tests/custom_cluster/test_session_expiration.py", line 171, in test_closing_idle_connection
> impalad.service.wait_for_metric_value(num_connections_metrics_name,
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/tests/common/impala_service.py", line 164, in wait_for_metric_value
> self.__metric_timeout_assert(metric_name, expected_value, timeout, value)
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/tests/common/impala_service.py", line 251, in __metric_timeout_assert
> assert 0, assert_string
> AssertionError: Metric impala.thrift-server.hiveserver2-frontend.connections-in-use did not reach value 0 in 5s. Actual value was '1'.
> Dumping debug webpages in JSON format...
> Dumped memz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/memz.json
> Dumped metrics JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/metrics.json
> Dumped queries JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/queries.json
> Dumped sessions JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/sessions.json
> Dumped threadz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/threadz.json
> Dumped rpcz JSON to $IMPALA_HOME/logs/metric_timeout_diags_20260121_11:32:20/json/rpcz.json
> Dumping minidumps for impalads/catalogds...
> Dumped minidump for Impalad PID 250362
> Dumped minidump for Catalogd PID 250268
> During handling of the above exception, another exception occurred:
> Traceback (most recent call last):
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/impala/hiveserver2.py", line 1195, in _execute
> open_transport(transport)
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/impala/hiveserver2.py", line 1259, in open_transport
> transport.open()
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/thrift/transport/TTransport.py", line 155, in open
> return self.__trans.open()
> File "/data0/jenkins/workspace/impala-cdw-master-core-asan-arm/repos/Impala/infra/python/env-gcc10.4.0-py3/lib/python3.8/site-packages/thrift/transport/TSocket.py", line 121, in open
> raise TTransportException(type=TTransportException.ALREADY_OPEN, message="already open")
> thrift.transport.TTransport.TTransportException: already open{noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-all-unsubscribe@impala.apache.org
For additional commands, e-mail: issues-all-help@impala.apache.org