(airflow) 39/48: [v3-2-test] Fix flaky K8s xcom tests on ARM runners hitting 120s pod-start timeout (#65598) (#65616)

[ra...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

vatsrahul1001 pushed a commit to branch v3-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 073d6dbee8209d9d02d54e7cea6f74d564b822c5
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Apr 21 20:17:38 2026 +0200

    [v3-2-test] Fix flaky K8s xcom tests on ARM runners hitting 120s pod-start timeout (#65598) (#65616)
    
    On ARM CI runners with a cold containerd cache, the first test in the K8s
    system suite that needs the xcom sidecar image (alpine) or the basic_pod
    template's image can exceed KubernetesPodOperator's 120s startup budget,
    producing a PodLaunchTimeoutException that surfaces as a generic
    "AirflowException: Pod ... returned a failure".
    
    Two changes:
    
    - basic_pod.yaml / test_full_pod_spec now use `ubuntu` instead of `perl`.
      The pod only runs `/bin/bash -c 'echo ... > /airflow/xcom/return.json'`,
      so any image with bash works, and `ubuntu` is already warmed by earlier
      tests in the same suite — no extra image pull needed.
    
    - Every real-cluster test that sets `do_xcom_push=True` now passes
      `startup_timeout_seconds=XCOM_STARTUP_TIMEOUT_SECONDS` (300s). Since
      pytest ordering is not guaranteed, whichever xcom test runs first has to
      absorb the one-time alpine sidecar pull; bumping the budget on all of
      them keeps the suite order-independent.
    
    Observed failure: apache/airflow actions run 24716106401, job 72301089157
    (K8S System:CeleryExecutor-3.12-v1.32.8-false, ARM). Both failing tests
    took exactly ~120s matching the default startup_timeout_seconds; pod
    events showed "Pulling image 'alpine' ..." with no "Successfully pulled"
    inside the 120s window.
    
    No production code change — the operator default of 120s is unchanged.
    (cherry picked from commit 25f07dce44b620b97757d00a24c4d2e36bf1a5a2)
    
    Co-authored-by: Jarek Potiuk <ja...@potiuk.com>
---
 kubernetes-tests/tests/kubernetes_tests/basic_pod.yaml     |  2 +-
 .../tests/kubernetes_tests/test_kubernetes_pod_operator.py | 14 ++++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/kubernetes-tests/tests/kubernetes_tests/basic_pod.yaml b/kubernetes-tests/tests/kubernetes_tests/basic_pod.yaml
index fff84d10972..5d79334179d 100644
--- a/kubernetes-tests/tests/kubernetes_tests/basic_pod.yaml
+++ b/kubernetes-tests/tests/kubernetes_tests/basic_pod.yaml
@@ -23,7 +23,7 @@ metadata:
 spec:
   containers:
     - name: base
-      image: perl
+      image: ubuntu
       command: ["/bin/bash"]
       args: ["-c", 'echo {\"hello\" : \"world\"} | cat > /airflow/xcom/return.json']
   restartPolicy: Never
diff --git a/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py
index 40bc4d2fbbb..a3067b0e8a1 100644
--- a/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py
+++ b/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py
@@ -51,6 +51,12 @@ from tests_common.test_utils.taskinstance import create_task_instance
 HOOK_CLASS = "airflow.providers.cncf.kubernetes.operators.pod.KubernetesHook"
 POD_MANAGER_CLASS = "airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager"
 
+# Longer than the operator's 120s default to absorb the first-pull latency of
+# the xcom sidecar image (alpine) on ARM runners with a cold containerd cache.
+# Whichever xcom test runs first has to pay that cost; giving all of them the
+# same budget keeps the tests order-independent.
+XCOM_STARTUP_TIMEOUT_SECONDS = 300
+
 
 def create_context(task) -> Context:
     from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
@@ -709,6 +715,7 @@ class TestKubernetesPodOperatorSystem:
             task_id=str(uuid4()),
             in_cluster=False,
             do_xcom_push=True,
+            startup_timeout_seconds=XCOM_STARTUP_TIMEOUT_SECONDS,
         )
         context = create_context(k)
         assert k.execute(context) == expected
@@ -753,6 +760,7 @@ class TestKubernetesPodOperatorSystem:
             labels=self.labels,
             pod_template_file=basic_pod_template.as_posix(),
             do_xcom_push=True,
+            startup_timeout_seconds=XCOM_STARTUP_TIMEOUT_SECONDS,
         )
 
         context = create_context(k)
@@ -775,6 +783,7 @@ class TestKubernetesPodOperatorSystem:
             in_cluster=False,
             pod_template_file=basic_pod_template.as_posix(),
             do_xcom_push=True,
+            startup_timeout_seconds=XCOM_STARTUP_TIMEOUT_SECONDS,
         )
 
         context = create_context(k)
@@ -814,6 +823,7 @@ class TestKubernetesPodOperatorSystem:
             pod_template_file=basic_pod_template.as_posix(),
             full_pod_spec=pod_spec,
             do_xcom_push=True,
+            startup_timeout_seconds=XCOM_STARTUP_TIMEOUT_SECONDS,
         )
 
         context = create_context(k)
@@ -842,7 +852,7 @@ class TestKubernetesPodOperatorSystem:
                 containers=[
                     k8s.V1Container(
                         name="base",
-                        image="perl",
+                        image="ubuntu",
                         command=["/bin/bash"],
                         args=["-c", 'echo {\\"hello\\" : \\"world\\"} | cat > /airflow/xcom/return.json'],
                         env=[k8s.V1EnvVar(name="env_name", value="value")],
@@ -858,7 +868,7 @@ class TestKubernetesPodOperatorSystem:
             full_pod_spec=pod_spec,
             do_xcom_push=True,
             on_finish_action=OnFinishAction.KEEP_POD,
-            startup_timeout_seconds=30,
+            startup_timeout_seconds=XCOM_STARTUP_TIMEOUT_SECONDS,
         )
 
         context = create_context(k)