You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by "em-eman (via GitHub)" <gi...@apache.org> on 2025/11/18 09:32:47 UTC
[D] Airflow 3.1.2 when high load api timeouts using kubernete executor pods [airflow]
GitHub user em-eman created a discussion: Airflow 3.1.2 when high load api timeouts using kubernete executor pods
It only happens when many dags are running otherwise pods are successfull.
I have following setting:
**scheduler : 2 replica counts
api/web: 2 replica counts
dag-processor : 1
worker: kubernetes executor**
```
025-11-18T09:17:52.005158Z] {{configuration.py:871}} DEBUG - Could not retrieve value from section database, for key sql_alchemy_engine_args. Skipping redaction of this conf.
[2025-11-18T09:17:52.005754Z] {{configuration.py:871}} DEBUG - Could not retrieve value from section database, for key sql_alchemy_conn_async. Skipping redaction of this conf.
{"timestamp":"2025-11-18T09:17:52.168665Z","level":"info","event":"Executing workload","workload":"ExecuteTask(token='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIwMTlhOTYzZi1lYjY1LTc0NGUtYTgzYi1kZDQ2ZTE4MjI5NDEiLCJqdGkiOiI4MTFhOTcyNzI0MzE0YjIzODRlYzQ3MTI5MjUxOTUxNSIsImlzcyI6Imh0dHBzOi8vYWlyZmxvdy13cy1uYXYtODY2Mi1wci53cy5uYXZpZ2FuY2UuY29tIiwiYXVkIjoidXJuOmFpcmZsb3cuYXBhY2hlLm9yZzp0YXNrIiwibmJmIjoxNzYzNDU3MzYzLCJleHAiOjE3NjM0NTc5NjMsImlhdCI6MTc2MzQ1NzM2M30.OB37xF8UgEjLB4FeDu6nno0RnmUOx8GWcf4Pvmj-N5Q', ti=TaskInstance(id=UUID('019a963f-eb65-744e-a83b-dd46e1822941'), dag_version_id=UUID('019a9525-3bf5-7c2f-9f58-4360d263ee9f'), task_id='unique_id_generator', dag_id='today_data_generator_sp_demo_c2_tailend_v2', run_id='manual__2025-11-18T09:16:01+00:00', try_number=1, map_index=-1, pool_slots=1, queue='kubernetes', priority_weight=5, executor_config=None, parent_context_carrier={}, context_carrier={}), dag_rel_path=PurePosixPath('plant/today_data_generator_v2.py'), bundle_info=BundleIn
fo(name='dags-folder', version=None), log_path='dag_id=today_data_generator_sp_demo_c2_tailend_v2/run_id=manual__2025-11-18T09:16:01+00:00/task_id=unique_id_generator/attempt=1.log', type='ExecuteTask')","logger":"__main__","filename":"execute_workload.py","lineno":56}
{"timestamp":"2025-11-18T09:17:52.169263Z","level":"info","event":"Connecting to server:","server":"http://airflow-web.ws-nav-8662-pr.svc.cluster.local/execution/","logger":"__main__","filename":"execute_workload.py","lineno":64}
{"timestamp":"2025-11-18T09:17:52.221962Z","level":"debug","event":"Connecting to execution API server","server":"http://airflow-web.ws-nav-8662-pr.svc.cluster.local/execution/","logger":"supervisor","filename":"supervisor.py","lineno":1920}
{"timestamp":"2025-11-18T09:18:12.240534Z","level":"warning","event":"Starting call to 'airflow.sdk.api.client.Client.request', this is the 1st time calling it.","logger":"airflow.sdk.api.client","filename":"before.py","lineno":42}
{"timestamp":"2025-11-18T09:18:32.959146Z","level":"warning","event":"Starting call to 'airflow.sdk.api.client.Client.request', this is the 2nd time calling it.","logger":"airflow.sdk.api.client","filename":"before.py","lineno":42}
{"timestamp":"2025-11-18T09:18:54.560692Z","level":"warning","event":"Starting call to 'airflow.sdk.api.client.Client.request', this is the 3rd time calling it.","logger":"airflow.sdk.api.client","filename":"before.py","lineno":42}
{"timestamp":"2025-11-18T09:19:16.095223Z","level":"warning","event":"Starting call to 'airflow.sdk.api.client.Client.request', this is the 4th time calling it.","logger":"airflow.sdk.api.client","filename":"before.py","lineno":42}
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
yield
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_transports/default.py", line 250, in handle_request
resp = self._pool.handle_request(req)
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection_pool.py", line 256, in handle_request
raise exc from None
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection_pool.py", line 236, in handle_request
response = connection.handle_request(
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection.py", line 101, in handle_request
raise exc
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection.py", line 78, in handle_request
stream = self._connect(request)
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection.py", line 124, in _connect
stream = self._network_backend.connect_tcp(**kwargs)
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_backends/sync.py", line 207, in connect_tcp
with map_exceptions(exc_map):
File "/usr/python/lib/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
raise to_exc(exc) from exc
httpcore.ConnectError: [Errno -3] Temporary failure in name resolution
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/python/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/python/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/execute_workload.py", line 125, in <module>
main()
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/execute_workload.py", line 121, in main
execute_workload(workload)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/execute_workload.py", line 66, in execute_workload
supervise(
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/supervisor.py", line 1928, in supervise
logger, log_file_descriptor = _configure_logging(log_path, client)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/supervisor.py", line 1843, in _configure_logging
with _remote_logging_conn(client):
File "/usr/python/lib/python3.10/contextlib.py", line 135, in __enter__
return next(self.gen)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/supervisor.py", line 898, in _remote_logging_conn
conn = _fetch_remote_logging_conn(conn_id, client)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/supervisor.py", line 862, in _fetch_remote_logging_conn
conn = client.connections.get(conn_id)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/api/client.py", line 361, in get
resp = self.client.get(f"connections/{conn_id}")
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 1053, in get
return self.request(
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 336, in wrapped_f
return copy(f, *args, **kw)
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 475, in __call__
do = self.iter(retry_state=retry_state)
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 376, in iter
result = action(retry_state)
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 418, in exc_check
raise retry_exc.reraise()
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 185, in reraise
raise self.last_attempt.result()
File "/usr/python/lib/python3.10/concurrent/futures/_base.py", line 451, in result
return self.__get_result()
File "/usr/python/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
raise self._exception
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 478, in __call__
result = fn(*args, **kwargs)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/api/client.py", line 885, in request
return super().request(*args, **kwargs)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 825, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 914, in send
response = self._send_handling_auth(
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 942, in _send_handling_auth
response = self._send_handling_redirects(
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 979, in _send_handling_redirects
response = self._send_single_request(request)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 1014, in _send_single_request
response = transport.handle_request(request)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_transports/default.py", line 249, in handle_request
with map_httpcore_exceptions():
File "/usr/python/lib/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectError: [Errno -3] Temporary failure in name resolution
```
GitHub link: https://github.com/apache/airflow/discussions/58420
----
This is an automatically sent email for commits@airflow.apache.org.
To unsubscribe, please send an email to: commits-unsubscribe@airflow.apache.org
Re: [D] Airflow 3.1.2 when high load api timeouts using kubernetes executor pods [airflow]
Posted by "em-eman (via GitHub)" <gi...@apache.org>.
GitHub user em-eman edited a discussion: Airflow 3.1.2 when high load api timeouts using kubernetes executor pods
It only happens when many dags are running otherwise pods are successfull.
I have following setting:
**scheduler : 2 replica counts
api/web: 2 replica counts
dag-processor : 1
worker: kubernetes executor**
```
025-11-18T09:17:52.005158Z] {{configuration.py:871}} DEBUG - Could not retrieve value from section database, for key sql_alchemy_engine_args. Skipping redaction of this conf.
[2025-11-18T09:17:52.005754Z] {{configuration.py:871}} DEBUG - Could not retrieve value from section database, for key sql_alchemy_conn_async. Skipping redaction of this conf.
{"timestamp":"2025-11-18T09:17:52.168665Z","level":"info","event":"Executing workload","workload":"ExecuteTask(token='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIwMTlhOTYzZi1lYjY1LTc0NGUtYTgzYi1kZDQ2ZTE4MjI5NDEiLCJqdGkiOiI4MTFhOTcyNzI0MzE0YjIzODRlYzQ3MTI5MjUxOTUxNSIsImlzcyI6Imh0dHBzOi8vYWlyZmxvdy13cy1uYXYtODY2Mi1wci53cy5uYXZpZ2FuY2UuY29tIiwiYXVkIjoidXJuOmFpcmZsb3cuYXBhY2hlLm9yZzp0YXNrIiwibmJmIjoxNzYzNDU3MzYzLCJleHAiOjE3NjM0NTc5NjMsImlhdCI6MTc2MzQ1NzM2M30.OB37xF8UgEjLB4FeDu6nno0RnmUOx8GWcf4Pvmj-N5Q', ti=TaskInstance(id=UUID('019a963f-eb65-744e-a83b-dd46e1822941'), dag_version_id=UUID('019a9525-3bf5-7c2f-9f58-4360d263ee9f'), task_id='unique_id_generator', dag_id='today_data_generator_sp_demo_c2_tailend_v2', run_id='manual__2025-11-18T09:16:01+00:00', try_number=1, map_index=-1, pool_slots=1, queue='kubernetes', priority_weight=5, executor_config=None, parent_context_carrier={}, context_carrier={}), dag_rel_path=PurePosixPath('plant/today_data_generator_v2.py'), bundle_info=BundleIn
fo(name='dags-folder', version=None), log_path='dag_id=today_data_generator_sp_demo_c2_tailend_v2/run_id=manual__2025-11-18T09:16:01+00:00/task_id=unique_id_generator/attempt=1.log', type='ExecuteTask')","logger":"__main__","filename":"execute_workload.py","lineno":56}
{"timestamp":"2025-11-18T09:17:52.169263Z","level":"info","event":"Connecting to server:","server":"http://airflow-web.ws-nav-8662-pr.svc.cluster.local/execution/","logger":"__main__","filename":"execute_workload.py","lineno":64}
{"timestamp":"2025-11-18T09:17:52.221962Z","level":"debug","event":"Connecting to execution API server","server":"http://airflow-web.ws-nav-8662-pr.svc.cluster.local/execution/","logger":"supervisor","filename":"supervisor.py","lineno":1920}
{"timestamp":"2025-11-18T09:18:12.240534Z","level":"warning","event":"Starting call to 'airflow.sdk.api.client.Client.request', this is the 1st time calling it.","logger":"airflow.sdk.api.client","filename":"before.py","lineno":42}
{"timestamp":"2025-11-18T09:18:32.959146Z","level":"warning","event":"Starting call to 'airflow.sdk.api.client.Client.request', this is the 2nd time calling it.","logger":"airflow.sdk.api.client","filename":"before.py","lineno":42}
{"timestamp":"2025-11-18T09:18:54.560692Z","level":"warning","event":"Starting call to 'airflow.sdk.api.client.Client.request', this is the 3rd time calling it.","logger":"airflow.sdk.api.client","filename":"before.py","lineno":42}
{"timestamp":"2025-11-18T09:19:16.095223Z","level":"warning","event":"Starting call to 'airflow.sdk.api.client.Client.request', this is the 4th time calling it.","logger":"airflow.sdk.api.client","filename":"before.py","lineno":42}
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
yield
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_transports/default.py", line 250, in handle_request
resp = self._pool.handle_request(req)
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection_pool.py", line 256, in handle_request
raise exc from None
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection_pool.py", line 236, in handle_request
response = connection.handle_request(
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection.py", line 101, in handle_request
raise exc
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection.py", line 78, in handle_request
stream = self._connect(request)
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_sync/connection.py", line 124, in _connect
stream = self._network_backend.connect_tcp(**kwargs)
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_backends/sync.py", line 207, in connect_tcp
with map_exceptions(exc_map):
File "/usr/python/lib/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/home/airflow/.local/lib/python3.10/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
raise to_exc(exc) from exc
httpcore.ConnectError: [Errno -3] Temporary failure in name resolution
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/python/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/python/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/execute_workload.py", line 125, in <module>
main()
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/execute_workload.py", line 121, in main
execute_workload(workload)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/execute_workload.py", line 66, in execute_workload
supervise(
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/supervisor.py", line 1928, in supervise
logger, log_file_descriptor = _configure_logging(log_path, client)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/supervisor.py", line 1843, in _configure_logging
with _remote_logging_conn(client):
File "/usr/python/lib/python3.10/contextlib.py", line 135, in __enter__
return next(self.gen)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/supervisor.py", line 898, in _remote_logging_conn
conn = _fetch_remote_logging_conn(conn_id, client)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/execution_time/supervisor.py", line 862, in _fetch_remote_logging_conn
conn = client.connections.get(conn_id)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/api/client.py", line 361, in get
resp = self.client.get(f"connections/{conn_id}")
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 1053, in get
return self.request(
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 336, in wrapped_f
return copy(f, *args, **kw)
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 475, in __call__
do = self.iter(retry_state=retry_state)
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 376, in iter
result = action(retry_state)
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 418, in exc_check
raise retry_exc.reraise()
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 185, in reraise
raise self.last_attempt.result()
File "/usr/python/lib/python3.10/concurrent/futures/_base.py", line 451, in result
return self.__get_result()
File "/usr/python/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
raise self._exception
File "/home/airflow/.local/lib/python3.10/site-packages/tenacity/__init__.py", line 478, in __call__
result = fn(*args, **kwargs)
File "/home/airflow/.local/lib/python3.10/site-packages/airflow/sdk/api/client.py", line 885, in request
return super().request(*args, **kwargs)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 825, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 914, in send
response = self._send_handling_auth(
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 942, in _send_handling_auth
response = self._send_handling_redirects(
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 979, in _send_handling_redirects
response = self._send_single_request(request)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_client.py", line 1014, in _send_single_request
response = transport.handle_request(request)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_transports/default.py", line 249, in handle_request
with map_httpcore_exceptions():
File "/usr/python/lib/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/home/airflow/.local/lib/python3.10/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectError: [Errno -3] Temporary failure in name resolution
```
GitHub link: https://github.com/apache/airflow/discussions/58420
----
This is an automatically sent email for commits@airflow.apache.org.
To unsubscribe, please send an email to: commits-unsubscribe@airflow.apache.org